accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject git commit: ACCUMULO-3263 oh, right... namenode write ops are slow when done one-at-a-time
Date Mon, 03 Nov 2014 20:26:59 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 43e0d5efb -> 8cdceebb6


ACCUMULO-3263 oh, right... namenode write ops are slow when done one-at-a-time


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8cdceebb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8cdceebb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8cdceebb

Branch: refs/heads/master
Commit: 8cdceebb6743edcba10ad64284cef07d14145c1f
Parents: 43e0d5e
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon Nov 3 15:10:29 2014 -0500
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon Nov 3 15:10:29 2014 -0500

----------------------------------------------------------------------
 .../accumulo/server/util/RandomizeVolumes.java  | 22 +++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/8cdceebb/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index d797392..6e5d8d2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -22,6 +22,8 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
 import java.io.IOException;
 import java.util.Map.Entry;
 
+import org.apache.accumulo.core.util.SimpleThreadPool;
+
 import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -67,7 +69,7 @@ public class RandomizeVolumes {
   }
 
   public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException,
AccumuloException, TableNotFoundException {
-    VolumeManager vm = VolumeManagerImpl.get();
+    final VolumeManager vm = VolumeManagerImpl.get();
     if (vm.getVolumes().size() < 2) {
       log.error("There are not enough volumes configured");
       return 1;
@@ -83,6 +85,7 @@ public class RandomizeVolumes {
       c.tableOperations().offline(tableName, true);
       log.info(tableName + " offline");
     }
+    SimpleThreadPool pool = new SimpleThreadPool(50, "directory maker");
     log.info("Rewriting entries for " + tableName);
     Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
     DIRECTORY_COLUMN.fetch(scanner);
@@ -106,18 +109,23 @@ public class RandomizeVolumes {
       Key key = entry.getKey();
       Mutation m = new Mutation(key.getRow());
 
-      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR
+ Path.SEPARATOR + tableId + Path.SEPARATOR
+      final String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR
+ ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
           + directory;
       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
       if (log.isTraceEnabled()) {
         log.trace("Replacing " + oldLocation + " with " + newLocation);
       }
       writer.addMutation(m);
-      try {
-        vm.mkdirs(new Path(newLocation));
-      } catch (IOException ex) {
-        // nevermind
-      }
+      pool.submit(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            vm.mkdirs(new Path(newLocation));
+          } catch (IOException ex) {
+            // nevermind
+          }
+        }
+      });
       count++;
     }
     writer.close();


Mime
View raw message