accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/3] git commit: ACCUMULO-3263 use volume chooser to select directory entries
Date Wed, 29 Oct 2014 14:45:35 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master efc75d1c5 -> 04e4312ed


ACCUMULO-3263 use volume chooser to select directory entries


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/309d9eb4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/309d9eb4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/309d9eb4

Branch: refs/heads/master
Commit: 309d9eb444f1aed1d63e48cea9966677d573d5a8
Parents: ca083a5
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Wed Oct 29 10:05:24 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Wed Oct 29 10:05:24 2014 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/server/util/Admin.java  |  2 +-
 .../accumulo/server/util/RandomizeVolumes.java  | 75 ++++++++++----------
 .../test/RewriteTabletDirectoriesIT.java        | 21 ++++--
 3 files changed, 56 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 5e7cb3e..bab42b5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -208,7 +208,7 @@ public class Admin {
       } else if (cl.getParsedCommand().equals("volumes")) {
         ListVolumesUsed.listVolumes(instance, principal, token);
       } else if (cl.getParsedCommand().equals("randomizeVolumes")) {
-        RandomizeVolumes.randomize(instance, new Credentials(principal, token), randomizeVolumesOpts.table);
+        rc = RandomizeVolumes.randomize(instance.getConnector(principal, token), randomizeVolumesOpts.table);
       } else {
         everything = cl.getParsedCommand().equals("stopAll");
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 7b3b6cc..afc3902 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -16,44 +16,48 @@
  */
 package org.apache.accumulo.server.util;
 
+import java.io.IOException;
 import java.util.Map.Entry;
 
+import org.apache.accumulo.server.security.SystemCredentials;
+
 import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.Credentials;
-import org.apache.accumulo.core.util.ColumnFQ;
 import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
 import org.apache.accumulo.server.tables.TableManager;
+import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 
 public class RandomizeVolumes {
   private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
 
-  public static void main(String[] args) {
+  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException
{
     ClientOnRequiredTable opts = new ClientOnRequiredTable();
     opts.parseArgs(RandomizeVolumes.class.getName(), args);
-    String principal = SystemCredentials.get().getPrincipal();
-    AuthenticationToken token = SystemCredentials.get().getToken();
+    Connector c;
+    if (opts.getToken() == null) {
+      SystemCredentials creds = SystemCredentials.get();
+      c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+    } else {
+      c = opts.getConnector();
+    }
     try {
-      int status = randomize(opts.getInstance(), new Credentials(principal, token), opts.tableName);
+      int status = randomize(c, opts.tableName);
       System.exit(status);
     } catch (Exception ex) {
       log.error(ex, ex);
@@ -61,42 +65,34 @@ public class RandomizeVolumes {
     }
   }
 
-  public static int randomize(Instance instance, Credentials credentials, String tableName)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
-    SiteConfiguration siteConfiguration = new ServerConfigurationFactory(instance).getSiteConfiguration();
-    String volumesConfig = siteConfiguration.get(Property.INSTANCE_VOLUMES);
-    if (null == volumesConfig || "".equals(volumesConfig)) {
-      log.error(Property.INSTANCE_VOLUMES.getKey() + " is not set in accumulo-site.xml");
+  public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException,
AccumuloException, TableNotFoundException {
+    VolumeManager vm = VolumeManagerImpl.get();
+    if (vm.getVolumes().size() < 2) {
+      log.error("There are not enough volumes configured");
       return 1;
     }
-    String[] volumes = volumesConfig.split(",");
-    if (volumes.length < 2) {
-      log.error("There are not enough volumes configured: " + volumesConfig);
-      return 2;
-    }
-    Connector c = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
     String tableId = c.tableOperations().tableIdMap().get(tableName);
     if (null == tableId) {
       log.error("Could not determine the table ID for table " + tableName);
-      return 3;
+      return 2;
     }
     TableState tableState = TableManager.getInstance().getTableState(tableId);
     if (TableState.OFFLINE != tableState) {
-      log.debug("Taking " + tableName + " offline");
+      log.info("Taking " + tableName + " offline");
       c.tableOperations().offline(tableName, true);
-      log.debug(tableName + " offline");
+      log.info(tableName + " offline");
     }
-    log.debug("Rewriting entries for " + tableName);
+    log.info("Rewriting entries for " + tableName);
     Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
-    final ColumnFQ DIRCOL = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
-    DIRCOL.fetch(scanner);
-    scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+    DIRECTORY_COLUMN.fetch(scanner);
+    scanner.setRange(TabletsSection.getRange(tableId));
     BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
     int count = 0;
     for (Entry<Key,Value> entry : scanner) {
       String oldLocation = entry.getValue().toString();
       String directory;
       if (oldLocation.contains(":")) {
-        String[] parts = oldLocation.split("/");
+        String[] parts = oldLocation.split(Path.SEPARATOR);
         String tableIdEntry = parts[parts.length - 2];
         if (!tableIdEntry.equals(tableId)) {
           log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId
+ "; skipping");
@@ -104,15 +100,22 @@ public class RandomizeVolumes {
         }
         directory = parts[parts.length - 1];
       } else {
-        directory = oldLocation.substring(1);
+        directory = oldLocation.substring(Path.SEPARATOR.length());
       }
-      Mutation m = new Mutation(entry.getKey().getRow());
-      String newLocation = volumes[count % volumes.length] + "/" + ServerConstants.TABLE_DIR
+ "/" + tableId + "/" + directory;
-      m.put(DIRCOL.getColumnFamily(), DIRCOL.getColumnQualifier(), new Value(newLocation.getBytes()));
+      Key key = entry.getKey();
+      Mutation m = new Mutation(key.getRow());
+      
+      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR
+ Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
+      m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes()));
       if (log.isTraceEnabled()) {
         log.trace("Replacing " + oldLocation + " with " + newLocation);
       }
       writer.addMutation(m);
+      try {
+        vm.mkdirs(new Path(newLocation));
+      } catch (IOException ex) {
+        // nevermind
+      }
       count++;
     }
     writer.close();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
index 4cd4a3e..d989789 100644
--- a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
@@ -25,7 +25,6 @@ import java.util.SortedSet;
 import java.util.TreeSet;
 
 import org.apache.accumulo.core.util.ColumnFQ;
-
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.security.TablePermission;
@@ -77,11 +76,16 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     Connector c = getConnector();
     c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME, TablePermission.WRITE);
     final String tableName = getUniqueNames(1)[0];
+    c.tableOperations().create(tableName);
+    BatchWriter bw = c.createBatchWriter(tableName, null);
     final SortedSet<Text> splits = new TreeSet<Text>();
     for (String split : "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(","))
{
       splits.add(new Text(split));
+      Mutation m = new Mutation(new Text(split));
+      m.put(new byte[]{}, new byte[]{}, new byte[]{});
+      bw.addMutation(m);
     }
-    c.tableOperations().create(tableName);
+    bw.close();
     c.tableOperations().addSplits(tableName, splits);
     
     BatchScanner scanner = c.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY,
1);
@@ -90,7 +94,7 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     String tableId = c.tableOperations().tableIdMap().get(tableName);
     scanner.setRanges(Collections.singletonList(MetadataSchema.TabletsSection.getRange(tableId)));
     // verify the directory entries are all on v1, make a few entries relative
-    BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, null);
+    bw = c.createBatchWriter(MetadataTable.NAME, null);
     int count = 0;
     for (Entry<Key,Value> entry : scanner) {
       assertTrue(entry.getValue().toString().contains(v1.toString()));
@@ -106,7 +110,7 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     assertEquals(splits.size() + 1, count);
     
     // This should fail: only one volume
-    assertEquals(2, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i",
c.getInstance().getInstanceName(), "-t", tableName).waitFor());
+    assertEquals(1, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i",
c.getInstance().getInstanceName(), "-t", tableName).waitFor());
 
     
     cluster.stop();
@@ -138,6 +142,13 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
       }
     }
     assertEquals(splits.size() + 1, v1Count + v2Count);
-    assertTrue(Math.abs(v1Count - v2Count) < 2);
+    assertTrue(Math.abs(v1Count - v2Count) < 10);
+    // verify we can read the old data
+    count = 0;
+    for (Entry<Key,Value> entry : c.createScanner(tableName, Authorizations.EMPTY))
{
+      assertTrue(splits.contains(entry.getKey().getRow()));
+      count++;
+    }
+    assertEquals(splits.size(), count);
   }
 }


Mime
View raw message