accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1494310 - in /accumulo/branches/ACCUMULO-118: core/src/main/java/org/apache/accumulo/core/conf/ server/src/main/java/org/apache/accumulo/server/fs/ server/src/main/java/org/apache/accumulo/server/master/recovery/ server/src/main/java/org/a...
Date Tue, 18 Jun 2013 21:20:07 GMT
Author: ecn
Date: Tue Jun 18 21:20:07 2013
New Revision: 1494310

URL: http://svn.apache.org/r1494310
Log:
ACCUMULO-118 use a configurable class to select volumes for new files

Added:
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
  (with props)
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
  (with props)
Modified:
    accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java

Modified: accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++ accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Tue Jun 18 21:20:07 2013
@@ -101,6 +101,8 @@ public enum Property {
   GENERAL_KERBEROS_PRINCIPAL("general.kerberos.principal", "", PropertyType.STRING, "Name
of the kerberos principal to use. _HOST will automatically be "
       + "replaced by the machines hostname in the hostname portion of the principal. Leave
blank if not using kerberoized hdfs"),
   GENERAL_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G", PropertyType.MEMORY,
"The maximum size of a message that can be sent to a tablet server."),
+  GENERAL_VOLUME_CHOOSER("general.volume.chooser", "org.apache.accumulo.server.fs.RandomVolumeChooser",
PropertyType.CLASSNAME, "The class that will be used to select which volume will be used to
create new files."),
+
   
   // properties that are specific to master server behavior
   MASTER_PREFIX("master.", null, PropertyType.PREFIX, "Properties in this category affect
the behavior of the master server"),

Added: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java?rev=1494310&view=auto
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
(added)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
Tue Jun 18 21:20:07 2013
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+import java.util.Random;
+
+public class RandomVolumeChooser implements VolumeChooser {
+  Random random = new Random();
+  
+  @Override
+  public String choose(String[] options) {
+    return options[random.nextInt(options.length)];
+  }
+
+}

Propchange: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java?rev=1494310&view=auto
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
(added)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
Tue Jun 18 21:20:07 2013
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+
+public interface VolumeChooser {
+  String choose(String[] options);
+}

Propchange: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
Tue Jun 18 21:20:07 2013
@@ -117,4 +117,6 @@ public interface VolumeManager {
 
   // forward to the appropriate FileSystem object
   ContentSummary getContentSummary(Path dir) throws IOException;
+
+  String choose(String[] options);
 }

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
Tue Jun 18 21:20:07 2013
@@ -58,12 +58,18 @@ public class VolumeManagerImpl implement
   Map<String, ? extends FileSystem> volumes;
   String defaultVolumes;
   AccumuloConfiguration conf;
+  VolumeChooser chooser;
   
   protected VolumeManagerImpl(Map<String, ? extends FileSystem> volumes, String defaultVolume,
AccumuloConfiguration conf) {
     this.volumes = volumes;
     this.defaultVolumes = defaultVolume;
     this.conf = conf;
     ensureSyncIsEnabled();
+    try {
+      this.getClass().getClassLoader().loadClass(conf.get(Property.GENERAL_VOLUME_CHOOSER));
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
   }
   
   public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws IOException
{
@@ -450,4 +456,9 @@ public class VolumeManagerImpl implement
     return getFileSystemByPath(dir).getContentSummary(dir);
   }
 
+  @Override
+  public String choose(String[] options) {
+    return chooser.choose(options);
+  }
+
 }

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
Tue Jun 18 21:20:07 2013
@@ -23,7 +23,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -119,10 +118,9 @@ public class RecoveryManager {
     log.info("Created zookeeper entry " + path + " with data " + work);
   }
   
-  Random random = new Random();
-  
   public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>>
walogs) throws IOException {
     boolean recoveryNeeded = false;
+    ;
     for (Collection<String> logs : walogs) {
       for (String walog : logs) {
         String hostFilename[] = walog.split("/", 2);
@@ -131,10 +129,8 @@ public class RecoveryManager {
         String parts[] = filename.split("/");
         String sortId = parts[parts.length - 1];
         // TODO: ACCUMULO-118: choose recovery directory with extension
-        String[] dirs = ServerConstants.getRecoveryDirs();
-        String recoveryDir = dirs[random.nextInt(dirs.length)];
-        String dest = recoveryDir + "/" + sortId;
-        log.debug("Recovering " + filename + " to " + dest + " using sortId " + sortId);
+        String dest = master.getFileSystem().choose(ServerConstants.getRecoveryDirs()) +
"/" + sortId;
+        log.debug("Recovering " + filename + " to " + dest);
         
         boolean sortQueued;
         synchronized (this) {

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
Tue Jun 18 21:20:07 2013
@@ -31,7 +31,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
@@ -232,19 +231,15 @@ public class DfsLogger {
     }
   }
   
-  // TODO: ACCUMULO-118
-  static final Random random = new Random();
-  
   public synchronized void open(String address) throws IOException {
     String filename = UUID.randomUUID().toString();
     logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
     
     log.debug("DfsLogger.open() begin");
-    String[] wals = ServerConstants.getWalDirs();
+    VolumeManager fs = conf.getFileSystem();
     
-    logPath = new Path(wals[random.nextInt(wals.length)] + "/" + logger + "/" + filename);
+    logPath = new Path(fs.choose(ServerConstants.getWalDirs()) + "/" + logger + "/" + filename);
     try {
-      VolumeManager fs = conf.getFileSystem();
       short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
       if (replication == 0)
         replication = fs.getDefaultReplication(logPath);

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
Tue Jun 18 21:20:07 2013
@@ -17,7 +17,6 @@
 package org.apache.accumulo.server.util;
 
 import java.io.IOException;
-import java.util.Random;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.util.UtilWaitThread;
@@ -33,27 +32,23 @@ public class TabletOperations {
   
   private static final Logger log = Logger.getLogger(TabletOperations.class);
   
-  private static final Random random = new Random();
-  
-  // TODO ACCUMULO-118 make the namespace selection pluggable
   public static String createTabletDirectory(VolumeManager fs, String tableId, Text endRow)
{
     String lowDirectory;
     
     UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
-    String[] tablesDirs = ServerConstants.getTablesDirs();
-    String randomNamespace = tablesDirs[random.nextInt(tablesDirs.length)];
+    String volume = fs.choose(ServerConstants.getTablesDirs());
     
     while (true) {
       try {
         if (endRow == null) {
           lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
-          Path lowDirectoryPath = new Path(randomNamespace + "/" + tableId + "/" + lowDirectory);
+          Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
           if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath))
             return lowDirectoryPath.makeQualified(fs.getFileSystemByPath(lowDirectoryPath)).toString();
           log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
         } else {
           lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX + namer.getNextName();
-          Path lowDirectoryPath = new Path(randomNamespace + "/" + tableId + "/" +  lowDirectory);
+          Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" +  lowDirectory);
           if (fs.exists(lowDirectoryPath))
             throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
           if (fs.mkdirs(lowDirectoryPath))
@@ -63,7 +58,7 @@ public class TabletOperations {
         log.warn(e);
       }
       
-      log.warn("Failed to create dir for tablet in table " + tableId + " in namespace " +
randomNamespace + " + will retry ...");
+      log.warn("Failed to create dir for tablet in table " + tableId + " in volume " + volume
+ " + will retry ...");
       UtilWaitThread.sleep(3000);
       
     }



Mime
View raw message