accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject [accumulo] branch master updated: Clean up Volume chooser ITs
Date Tue, 22 Jan 2019 23:15:44 GMT
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 7db5a0d  Clean up Volume chooser ITs
7db5a0d is described below

commit 7db5a0d8b4ca9fb3abde81c4f1caa6ff24af5685
Author: Mike Miller <mmiller@apache.org>
AuthorDate: Tue Jan 22 18:14:06 2019 -0500

    Clean up Volume chooser ITs
---
 .../org/apache/accumulo/test/VolumeChooserIT.java  | 107 +++++-------------
 .../java/org/apache/accumulo/test/VolumeIT.java    | 124 ++++++++-------------
 2 files changed, 76 insertions(+), 155 deletions(-)

diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
index 620ce4c..cbbcd84 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
@@ -64,7 +64,8 @@ public class VolumeChooserIT extends ConfigurableMacBase {
   private File volDirBase;
   @SuppressWarnings("unused")
   private Path v1, v2, v3, v4;
-  private static String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+  public static String[] alpha_rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z"
+      .split(",");
   private String namespace1;
   private String namespace2;
   private String systemPreferredVolumes;
@@ -127,28 +128,28 @@ public class VolumeChooserIT extends ConfigurableMacBase {
       throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
     // Add 10 splits to the table
     SortedSet<Text> partitions = new TreeSet<>();
-    for (String s : rows)
+    for (String s : alpha_rows)
       partitions.add(new Text(s));
     accumuloClient.tableOperations().addSplits(tableName, partitions);
   }
 
   public static void writeAndReadData(AccumuloClient accumuloClient, String tableName)
       throws Exception {
-    writeDataToTable(accumuloClient, tableName);
+    writeDataToTable(accumuloClient, tableName, alpha_rows);
 
     // Write the data to disk, read it back
     accumuloClient.tableOperations().flush(tableName, null, null, true);
     try (Scanner scanner = accumuloClient.createScanner(tableName, Authorizations.EMPTY))
{
       int i = 0;
       for (Entry<Key,Value> entry : scanner) {
-        assertEquals("Data read is not data written", rows[i++],
+        assertEquals("Data read is not data written", alpha_rows[i++],
             entry.getKey().getRow().toString());
       }
     }
   }
 
-  public static void writeDataToTable(AccumuloClient accumuloClient, String tableName)
-      throws Exception {
+  public static void writeDataToTable(AccumuloClient accumuloClient, String tableName,
+      String[] rows) throws Exception {
     // Write some data to the table
     BatchWriter bw = accumuloClient.createBatchWriter(tableName, new BatchWriterConfig());
     for (String s : rows) {
@@ -259,28 +260,18 @@ public class VolumeChooserIT extends ConfigurableMacBase {
     log.info("Starting twoTablesPreferredVolumeChooser");
 
     // Create namespace
-    try (AccumuloClient accumuloClient = createClient()) {
-      accumuloClient.namespaceOperations().create(namespace1);
-
+    try (AccumuloClient c = createClient()) {
       // Set properties on the namespace
       // namespace 1 -> v2
-      accumuloClient.namespaceOperations().setProperty(namespace1,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
-      accumuloClient.namespaceOperations().setProperty(namespace1,
-          PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v2.toString());
+      configureNamespace(c, PreferredVolumeChooser.class.getName(), v2.toString(), namespace1);
 
       // Create table1 on namespace1
-      verifyVolumesForWritesToNewTable(accumuloClient, namespace1, v2.toString());
+      verifyVolumesForWritesToNewTable(c, namespace1, v2.toString());
 
-      accumuloClient.namespaceOperations().create(namespace2);
-      // Set properties on the namespace
-      accumuloClient.namespaceOperations().setProperty(namespace2,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
-      accumuloClient.namespaceOperations().setProperty(namespace2,
-          PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
+      configureNamespace(c, PreferredVolumeChooser.class.getName(), v1.toString(), namespace2);
 
       // Create table2 on namespace2
-      verifyVolumesForWritesToNewTable(accumuloClient, namespace2, v1.toString());
+      verifyVolumesForWritesToNewTable(c, namespace2, v1.toString());
     }
   }
 
@@ -292,72 +283,34 @@ public class VolumeChooserIT extends ConfigurableMacBase {
 
     // Create namespace
     try (AccumuloClient accumuloClient = createClient()) {
-      accumuloClient.namespaceOperations().create(namespace1);
-
-      // Set properties on the namespace
-      accumuloClient.namespaceOperations().setProperty(namespace1,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
-
-      // Create table1 on namespace1
-      String tableName = namespace1 + ".1";
-      accumuloClient.tableOperations().create(tableName);
-      Table.ID tableID = Table.ID.of(accumuloClient.tableOperations().tableIdMap().get(tableName));
-
-      // Add 10 splits to the table
-      addSplits(accumuloClient, tableName);
-      // Write some data to the table
-      writeAndReadData(accumuloClient, tableName);
-      // Verify the new files are written to the Volumes specified
-
-      verifyVolumes(accumuloClient, TabletsSection.getRange(tableID), v1 + "," + v2 + ","
+ v4);
+      createAndVerify(accumuloClient, namespace1, v1 + "," + v2 + "," + v4);
+      createAndVerify(accumuloClient, namespace2, v1 + "," + v2 + "," + v4);
+    }
+  }
 
-      accumuloClient.namespaceOperations().create(namespace2);
+  private void createAndVerify(AccumuloClient accumuloClient, String ns, String expectedVolumes)
+      throws Exception {
+    accumuloClient.namespaceOperations().create(ns);
 
-      // Set properties on the namespace
-      accumuloClient.namespaceOperations().setProperty(namespace2,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
+    // Set properties on the namespace
+    accumuloClient.namespaceOperations().setProperty(ns, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
+        RandomVolumeChooser.class.getName());
 
-      // Create table2 on namespace2
-      String tableName2 = namespace2 + ".1";
-      accumuloClient.tableOperations().create(tableName2);
-      Table.ID tableID2 = Table.ID
-          .of(accumuloClient.tableOperations().tableIdMap().get(tableName2));
-
-      // / Add 10 splits to the table
-      addSplits(accumuloClient, tableName2);
-      // Write some data to the table
-      writeAndReadData(accumuloClient, tableName2);
-      // Verify the new files are written to the Volumes specified
-      verifyVolumes(accumuloClient, TabletsSection.getRange(tableID2), v1 + "," + v2 + ","
+ v4);
-    }
+    verifyVolumesForWritesToNewTable(accumuloClient, ns, expectedVolumes);
   }
 
-  // Test that uses two tables with 10 split points each. The first uses the RandomVolumeChooser
and
-  // the second uses the
-  // StaticVolumeChooser to choose volumes.
+  // Test that uses 2 tables with 10 split points each. The first uses the RandomVolumeChooser
and
+  // the second uses the StaticVolumeChooser to choose volumes.
   @Test
   public void twoTablesDiffChoosers() throws Exception {
     log.info("Starting twoTablesDiffChoosers");
 
     // Create namespace
-    try (AccumuloClient accumuloClient = createClient()) {
-      accumuloClient.namespaceOperations().create(namespace1);
-
-      // Set properties on the namespace
-      accumuloClient.namespaceOperations().setProperty(namespace1,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
-
-      // Create table1 on namespace1
-      verifyVolumesForWritesToNewTable(accumuloClient, namespace1, v1 + "," + v2 + "," +
v4);
-      accumuloClient.namespaceOperations().create(namespace2);
-
-      accumuloClient.namespaceOperations().setProperty(namespace2,
-          PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
-      accumuloClient.namespaceOperations().setProperty(namespace2,
-          PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
-
+    try (AccumuloClient c = createClient()) {
+      createAndVerify(c, namespace1, v1 + "," + v2 + "," + v4);
+      configureNamespace(c, PreferredVolumeChooser.class.getName(), v1.toString(), namespace2);
       // Create table2 on namespace2
-      verifyVolumesForWritesToNewTable(accumuloClient, namespace2, v1.toString());
+      verifyVolumesForWritesToNewTable(c, namespace2, v1.toString());
     }
   }
 
@@ -383,7 +336,7 @@ public class VolumeChooserIT extends ConfigurableMacBase {
       accumuloClient.tableOperations().create(tableName);
 
       VolumeChooserIT.addSplits(accumuloClient, tableName);
-      VolumeChooserIT.writeDataToTable(accumuloClient, tableName);
+      VolumeChooserIT.writeDataToTable(accumuloClient, tableName, alpha_rows);
       // should only go to v2 as per configuration in configure()
       VolumeChooserIT.verifyWaLogVolumes(accumuloClient, new Range(), v2.toString());
     }
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index c9626f6..4347d82 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@ -39,7 +39,6 @@ import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
@@ -85,10 +84,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
 public class VolumeIT extends ConfigurableMacBase {
 
-  private static final Text EMPTY = new Text();
-  private static final Value EMPTY_VALUE = new Value(new byte[] {});
   private File volDirBase;
-  private Path v1, v2;
+  private Path v1, v2, v3;
+  private List<String> expected = new ArrayList<>();
 
   @Override
   protected int defaultTimeoutSeconds() {
@@ -104,6 +102,13 @@ public class VolumeIT extends ConfigurableMacBase {
     File v2f = new File(volDirBase, "v2");
     v1 = new Path("file://" + v1f.getAbsolutePath());
     v2 = new Path("file://" + v2f.getAbsolutePath());
+    File v3f = new File(volDirBase, "v3");
+    v3 = new Path("file://" + v3f.getAbsolutePath());
+    // setup expected rows
+    for (int i = 0; i < 100; i++) {
+      String row = String.format("%06d", i * 100 + 3);
+      expected.add(row + ":cf1:cq1:1");
+    }
 
     // Run MAC on two locations in the local file system
     URI v1Uri = v1.toUri();
@@ -131,20 +136,13 @@ public class VolumeIT extends ConfigurableMacBase {
         partitions.add(new Text(s));
       accumuloClient.tableOperations().addSplits(tableName, partitions);
       // scribble over the splits
-      BatchWriter bw = accumuloClient.createBatchWriter(tableName, new BatchWriterConfig());
-      String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
-      for (String s : rows) {
-        Mutation m = new Mutation(new Text(s));
-        m.put(EMPTY, EMPTY, EMPTY_VALUE);
-        bw.addMutation(m);
-      }
-      bw.close();
+      VolumeChooserIT.writeDataToTable(accumuloClient, tableName, VolumeChooserIT.alpha_rows);
       // write the data to disk, read it back
       accumuloClient.tableOperations().flush(tableName, null, null, true);
       try (Scanner scanner = accumuloClient.createScanner(tableName, Authorizations.EMPTY))
{
         int i = 0;
         for (Entry<Key,Value> entry : scanner) {
-          assertEquals(rows[i++], entry.getKey().getRow().toString());
+          assertEquals(VolumeChooserIT.alpha_rows[i++], entry.getKey().getRow().toString());
         }
       }
       // verify the new files are written to the different volumes
@@ -210,9 +208,7 @@ public class VolumeIT extends ConfigurableMacBase {
       BatchWriter bw = accumuloClient.createBatchWriter(tableName, new BatchWriterConfig());
 
       // create two files in each tablet
-
-      String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
-      for (String s : rows) {
+      for (String s : VolumeChooserIT.alpha_rows) {
         Mutation m = new Mutation(s);
         m.put("cf1", "cq1", "1");
         bw.addMutation(m);
@@ -222,7 +218,7 @@ public class VolumeIT extends ConfigurableMacBase {
       bw.flush();
       accumuloClient.tableOperations().flush(tableName, null, null, true);
 
-      for (String s : rows) {
+      for (String s : VolumeChooserIT.alpha_rows) {
         Mutation m = new Mutation(s);
         m.put("cf1", "cq1", "2");
         bw.addMutation(m);
@@ -284,21 +280,9 @@ public class VolumeIT extends ConfigurableMacBase {
   public void testAddVolumes() throws Exception {
     try (AccumuloClient client = createClient()) {
       String[] tableNames = getUniqueNames(2);
-
-      // grab this before shutting down cluster
-      String uuid = client.getInstanceID();
-
-      verifyVolumesUsed(client, tableNames[0], false, v1, v2);
-
-      assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
-      cluster.stop();
-
       PropertiesConfiguration conf = new PropertiesConfiguration();
-      conf.load(cluster.getAccumuloPropertiesPath());
 
-      File v3f = new File(volDirBase, "v3");
-      assertTrue(v3f.mkdir() || v3f.isDirectory());
-      Path v3 = new Path("file://" + v3f.getAbsolutePath());
+      String uuid = verifyAndShutdownCluster(client, conf, tableNames[0]);
 
       conf.setProperty(Property.INSTANCE_VOLUMES.getKey(), v1 + "," + v2 + "," + v3);
       conf.save(cluster.getAccumuloPropertiesPath());
@@ -306,14 +290,7 @@ public class VolumeIT extends ConfigurableMacBase {
       // initialize volume
       assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 
-      // check that all volumes are initialized
-      for (Path volumePath : Arrays.asList(v1, v2, v3)) {
-        FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
-        Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
-        FileStatus[] iids = fs.listStatus(vp);
-        assertEquals(1, iids.length);
-        assertEquals(uuid, iids[0].getPath().getName());
-      }
+      checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid);
 
       // start cluster and verify that new volume is used
       cluster.start();
@@ -322,26 +299,28 @@ public class VolumeIT extends ConfigurableMacBase {
     }
   }
 
-  @Test
-  public void testNonConfiguredVolumes() throws Exception {
+  // grab uuid before shutting down cluster
+  private String verifyAndShutdownCluster(AccumuloClient c, PropertiesConfiguration conf,
+      String tableName) throws Exception {
+    String uuid = c.getInstanceID();
 
-    String[] tableNames = getUniqueNames(2);
+    verifyVolumesUsed(c, tableName, false, v1, v2);
 
-    // grab this before shutting down cluster
-    try (AccumuloClient client = createClient()) {
-      String uuid = client.getInstanceID();
+    assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+    cluster.stop();
 
-      verifyVolumesUsed(client, tableNames[0], false, v1, v2);
+    conf.load(cluster.getAccumuloPropertiesPath());
+    return uuid;
+  }
 
-      assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
-      cluster.stop();
+  @Test
+  public void testNonConfiguredVolumes() throws Exception {
 
-      PropertiesConfiguration conf = new PropertiesConfiguration();
-      conf.load(cluster.getAccumuloPropertiesPath());
+    String[] tableNames = getUniqueNames(2);
+    PropertiesConfiguration conf = new PropertiesConfiguration();
 
-      File v3f = new File(volDirBase, "v3");
-      assertTrue(v3f.mkdir() || v3f.isDirectory());
-      Path v3 = new Path("file://" + v3f.getAbsolutePath());
+    try (AccumuloClient client = createClient()) {
+      String uuid = verifyAndShutdownCluster(client, conf, tableNames[0]);
 
       conf.setProperty(Property.INSTANCE_VOLUMES.getKey(), v2 + "," + v3);
       conf.save(cluster.getAccumuloPropertiesPath());
@@ -349,27 +328,12 @@ public class VolumeIT extends ConfigurableMacBase {
       // initialize volume
       assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 
-      // check that all volumes are initialized
-      for (Path volumePath : Arrays.asList(v1, v2, v3)) {
-        FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
-        Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
-        FileStatus[] iids = fs.listStatus(vp);
-        assertEquals(1, iids.length);
-        assertEquals(uuid, iids[0].getPath().getName());
-      }
+      checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid);
 
       // start cluster and verify that new volume is used
       cluster.start();
 
-      // Make sure we can still read the tables (tableNames[0] is very likely to have a file
still
-      // on
-      // v1)
-      List<String> expected = new ArrayList<>();
-      for (int i = 0; i < 100; i++) {
-        String row = String.format("%06d", i * 100 + 3);
-        expected.add(row + ":cf1:cq1:1");
-      }
-
+      // verify we can still read the tables (tableNames[0] is likely to have a file still
on v1)
       verifyData(expected, client.createScanner(tableNames[0], Authorizations.EMPTY));
 
       // v1 should not have any data for tableNames[1]
@@ -377,9 +341,19 @@ public class VolumeIT extends ConfigurableMacBase {
     }
   }
 
-  private void writeData(String tableName, AccumuloClient client)
-      throws AccumuloException, AccumuloSecurityException, TableExistsException,
-      TableNotFoundException, MutationsRejectedException {
+  // check that all volumes are initialized
+  private void checkVolumesInitialized(List<Path> volumes, String uuid) throws Exception
{
+    for (Path volumePath : volumes) {
+      FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+      Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+      FileStatus[] iids = fs.listStatus(vp);
+      assertEquals(1, iids.length);
+      assertEquals(uuid, iids[0].getPath().getName());
+    }
+  }
+
+  private void writeData(String tableName, AccumuloClient client) throws AccumuloException,
+      AccumuloSecurityException, TableExistsException, TableNotFoundException {
     TreeSet<Text> splits = new TreeSet<>();
     for (int i = 1; i < 100; i++) {
       splits.add(new Text(String.format("%06d", i * 100)));
@@ -402,12 +376,6 @@ public class VolumeIT extends ConfigurableMacBase {
   private void verifyVolumesUsed(AccumuloClient client, String tableName, boolean shouldExist,
       Path... paths) throws Exception {
 
-    List<String> expected = new ArrayList<>();
-    for (int i = 0; i < 100; i++) {
-      String row = String.format("%06d", i * 100 + 3);
-      expected.add(row + ":cf1:cq1:1");
-    }
-
     if (!client.tableOperations().exists(tableName)) {
       assertFalse(shouldExist);
 


Mime
View raw message