accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/3] git commit: ACCUMULO-3263 oh, right... namenode write ops are slow when done one-at-a-time
Date Tue, 04 Nov 2014 17:07:31 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 8cdceebb6 -> 7196b0c58


ACCUMULO-3263 oh, right... namenode write ops are slow when done one-at-a-time


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a48878f3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a48878f3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a48878f3

Branch: refs/heads/master
Commit: a48878f34b53b1292200ab4d0f98c24308b979f9
Parents: 933e39d
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon Nov 3 15:10:29 2014 -0500
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon Nov 3 15:26:24 2014 -0500

----------------------------------------------------------------------
 .../accumulo/server/util/RandomizeVolumes.java  | 22 +++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/a48878f3/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 8924c85..4114ed2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -22,6 +22,8 @@ import java.io.IOException;
 import java.util.Map.Entry;
 
 import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+
 import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -67,7 +69,7 @@ public class RandomizeVolumes {
   }
 
   public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException,
AccumuloException, TableNotFoundException {
-    VolumeManager vm = VolumeManagerImpl.get();
+    final VolumeManager vm = VolumeManagerImpl.get();
     if (vm.getVolumes().size() < 2) {
       log.error("There are not enough volumes configured");
       return 1;
@@ -83,6 +85,7 @@ public class RandomizeVolumes {
       c.tableOperations().offline(tableName, true);
       log.info(tableName + " offline");
     }
+    SimpleThreadPool pool = new SimpleThreadPool(50, "directory maker");
     log.info("Rewriting entries for " + tableName);
     Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
     DIRECTORY_COLUMN.fetch(scanner);
@@ -106,17 +109,22 @@ public class RandomizeVolumes {
       Key key = entry.getKey();
       Mutation m = new Mutation(key.getRow());
       
-      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR
+ Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
+      final String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR
+ ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
       if (log.isTraceEnabled()) {
         log.trace("Replacing " + oldLocation + " with " + newLocation);
       }
       writer.addMutation(m);
-      try {
-        vm.mkdirs(new Path(newLocation));
-      } catch (IOException ex) {
-        // nevermind
-      }
+      pool.submit(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            vm.mkdirs(new Path(newLocation));
+          } catch (IOException ex) {
+            // nevermind
+          }
+        }
+      });
       count++;
     }
     writer.close();


Mime
View raw message