accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch master updated: Add endRow to VolumeChooserEnvironment (#524)
Date Mon, 04 Feb 2019 23:00:07 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 050dec2  Add endRow to VolumeChooserEnvironment (#524)
050dec2 is described below

commit 050dec2003a786ea014c994a38e180a82b997c0d
Author: Adam Lerman <alerman@gmail.com>
AuthorDate: Mon Feb 4 17:53:40 2019 -0500

    Add endRow to VolumeChooserEnvironment (#524)
---
 .../accumulo/server/fs/VolumeChooserEnvironment.java  | 19 ++++++++++++++++++-
 .../org/apache/accumulo/server/fs/VolumeUtil.java     |  3 ++-
 .../accumulo/server/util/MetadataTableUtil.java       |  5 +++--
 .../apache/accumulo/server/util/RandomizeVolumes.java |  5 ++++-
 .../accumulo/server/fs/PerTableVolumeChooserTest.java |  3 ++-
 .../server/fs/PreferredVolumeChooserTest.java         |  3 ++-
 .../accumulo/server/fs/VolumeManagerImplTest.java     |  2 +-
 .../main/java/org/apache/accumulo/master/Master.java  |  3 ++-
 .../apache/accumulo/master/TabletGroupWatcher.java    |  3 ++-
 .../accumulo/master/tableOps/create/ChooseDir.java    |  2 +-
 .../tableOps/tableImport/PopulateMetadataTable.java   |  9 +++++----
 .../master/tableOps/tableImport/ImportTableTest.java  |  4 ++--
 .../org/apache/accumulo/tserver/tablet/Tablet.java    |  2 +-
 13 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
index 985e29e..a1a4f3e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
@@ -20,6 +20,7 @@ import java.util.Objects;
 
 import org.apache.accumulo.core.clientImpl.Table;
 import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
 
 public class VolumeChooserEnvironment {
 
@@ -35,17 +36,33 @@ public class VolumeChooserEnvironment {
   private final ServerContext context;
   private final ChooserScope scope;
   private final Table.ID tableId;
+  private final Text endRow;
 
   public VolumeChooserEnvironment(ChooserScope scope, ServerContext context) {
     this.context = context;
     this.scope = Objects.requireNonNull(scope);
     this.tableId = null;
+    this.endRow = null;
   }
 
-  public VolumeChooserEnvironment(Table.ID tableId, ServerContext context) {
+  public VolumeChooserEnvironment(Table.ID tableId, Text endRow, ServerContext context) {
     this.context = context;
     this.scope = ChooserScope.TABLE;
     this.tableId = Objects.requireNonNull(tableId);
+    this.endRow = endRow;
+
+  }
+
+  /**
+   * The end row of the tablet for which a volume is being chosen. Only call this when the
scope is
+   * TABLE
+   *
+   * @since 2.0.0
+   */
+  public Text getEndRow() {
+    if (scope != ChooserScope.TABLE)
+      throw new IllegalStateException("Can only request end row for tables, not for " + scope);
+    return endRow;
   }
 
   public Table.ID getTableId() {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index f6c2dd0..0653134 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -273,7 +273,8 @@ public class VolumeUtil {
     }
 
     VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(extent.getTableId(),
-        context);
+        extent.getEndRow(), context);
+
     Path newDir = new Path(vm.choose(chooserEnv, ServerConstants.getBaseUris(context))
         + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + dir.getParent().getName()
         + Path.SEPARATOR + dir.getName());
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index b5baa50..0e8beb2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -914,7 +914,8 @@ public class MetadataTableUtil {
         Key k = entry.getKey();
         Mutation m = new Mutation(k.getRow());
         m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
-        VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+        VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId,
+            new KeyExtent(k.getRow(), (Text) null).getEndRow(), context);
         String dir = volumeManager.choose(chooserEnv, ServerConstants.getBaseUris(context))
             + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + new
String(
                 FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
@@ -1023,7 +1024,7 @@ public class MetadataTableUtil {
    */
   public static void createReplicationTable(ServerContext context) {
 
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ReplicationTable.ID,
+    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ReplicationTable.ID,
null,
         context);
     String dir = context.getVolumeManager().choose(chooserEnv, ServerConstants.getBaseUris(context))
         + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 94fbde1..a16b2bf 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.clientImpl.Table;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.metadata.MetadataTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
@@ -43,6 +44,7 @@ import org.apache.accumulo.server.cli.ServerUtilOnRequiredTable;
 import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -105,7 +107,8 @@ public class RandomizeVolumes {
       Key key = entry.getKey();
       Mutation m = new Mutation(key.getRow());
 
-      VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+      VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId,
+          new KeyExtent(key.getRow(), (Text) null).getEndRow(), context);
       final String newLocation = vm.choose(chooserEnv, ServerConstants.getBaseUris(context))
           + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
           + directory;
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
index 89e8f9e..eaec3cb 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
@@ -78,7 +78,8 @@ public class PerTableVolumeChooserTest {
   }
 
   private VolumeChooser getTableDelegate() {
-    VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"),
null);
+    VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"),
null,
+        null);
     return chooser.getDelegateChooser(env);
   }
 
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
index 076b29a..f1edb9c 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
@@ -74,7 +74,8 @@ public class PreferredVolumeChooserTest {
   }
 
   private String[] chooseForTable() {
-    VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"),
null);
+    VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"),
null,
+        null);
     return chooser.getPreferredVolumes(env, ALL_OPTIONS);
   }
 
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
index 8a3c7e7..a3d704d 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
@@ -138,7 +138,7 @@ public class VolumeManagerImplTest {
     thrown.expect(RuntimeException.class);
     VolumeManager vm = VolumeManagerImpl.get(conf, hadoopConf);
     VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Table.ID.of("sometable"),
-        null);
+        null, null);
     String choice = vm.choose(chooserEnv, volumes.toArray(new String[0]));
     assertTrue("shouldn't see invalid options from misbehaving chooser.", volumes.contains(choice));
   }
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index f130e0c..8005af9 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -286,7 +286,8 @@ public class Master
     if (!zoo.exists(dirZPath)) {
       Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
       if (fs.exists(oldPath)) {
-        VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(RootTable.ID,
context);
+        VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(RootTable.ID,
+            RootTable.EXTENT.getEndRow(), context);
         String newPath = fs.choose(chooserEnv, ServerConstants.getBaseUris(context))
             + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
         fs.mkdirs(new Path(newPath));
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index 4c4fe8a..0a1437d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -660,7 +660,8 @@ abstract class TabletGroupWatcher extends Daemon {
         // Recreate the default tablet to hold the end of the table
         Master.log.debug("Recreating the last tablet to point to {}", extent.getPrevEndRow());
         VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(extent.getTableId(),
-            master.getContext());
+            extent.getEndRow(), master.getContext());
+
         String tdir = master.getFileSystem().choose(chooserEnv,
             ServerConstants.getBaseUris(master.getContext())) + Constants.HDFS_TABLES_DIR
             + Path.SEPARATOR + extent.getTableId() + Constants.DEFAULT_TABLET_LOCATION;
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
index bf3e4e0..0c401d9 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
@@ -54,7 +54,7 @@ class ChooseDir extends MasterRepo {
     // Constants.DEFAULT_TABLET_LOCATION has a leading slash prepended to it so we don't
need to add
     // one here
 
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.getTableId(),
+    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.getTableId(),
null,
         master.getContext());
 
     String baseDir = master.getFileSystem().choose(chooserEnv,
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
index ca2c968..156dd95 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
@@ -152,7 +152,7 @@ class PopulateMetadataTable extends MasterRepo {
                   UTF_8);
 
               // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
-              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+              String absolutePath = getClonedTabletDir(master, endRow, tableDirs, tabletDir);
 
               m = new Mutation(metadataRow);
               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -170,7 +170,7 @@ class PopulateMetadataTable extends MasterRepo {
                   UTF_8);
 
               // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
-              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+              String absolutePath = getClonedTabletDir(master, endRow, tableDirs, tabletDir);
 
               m = new Mutation(metadataRow);
               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -217,9 +217,10 @@ class PopulateMetadataTable extends MasterRepo {
    *
    * @return An absolute, unique path for the imported table
    */
-  protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir)
{
+  protected String getClonedTabletDir(Master master, Text endRow, String[] tableDirs,
+      String tabletDir) {
     // We can try to spread out the tablet dirs across all volumes
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.tableId,
+    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.tableId,
endRow,
         master.getContext());
     String tableDir = master.getFileSystem().choose(chooserEnv, tableDirs);
 
diff --git a/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
b/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
index e9577a7..313adda 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
@@ -43,7 +43,7 @@ public class ImportTableTest {
     EasyMock.expect(master.getContext()).andReturn(null);
     EasyMock.expect(master.getFileSystem()).andReturn(volumeManager);
     // Choose the 2nd element
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(iti.tableId, null);
+    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(iti.tableId, null,
null);
     EasyMock.expect(volumeManager.choose(EasyMock.eq(chooserEnv), EasyMock.eq(tableDirs)))
         .andReturn(tableDirs[1]);
 
@@ -51,7 +51,7 @@ public class ImportTableTest {
 
     PopulateMetadataTable pmt = new PopulateMetadataTable(iti);
     assertEquals(tableDirs[1] + "/" + iti.tableId + "/" + tabletDir,
-        pmt.getClonedTabletDir(master, tableDirs, tabletDir));
+        pmt.getClonedTabletDir(master, null, tableDirs, tabletDir));
 
     EasyMock.verify(master, volumeManager);
   }
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 1720343..5730bed 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2861,7 +2861,7 @@ public class Tablet {
     String lowDirectory;
 
     UniqueNameAllocator namer = context.getUniqueNameAllocator();
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, endRow, context);
     String volume = fs.choose(chooserEnv, ServerConstants.getBaseUris(context))
         + Constants.HDFS_TABLES_DIR + Path.SEPARATOR;
 


Mime
View raw message