accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [04/23] git commit: ACCUMULO-2061 Fix the rest of server-base
Date Tue, 11 Mar 2014 18:25:13 GMT
ACCUMULO-2061 Fix the rest of server-base


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e613f710
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e613f710
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e613f710

Branch: refs/heads/ACCUMULO-2061
Commit: e613f7105e3a0828096909535147b4bc80a265f6
Parents: 3b3ccbb
Author: Josh Elser <elserj@apache.org>
Authored: Fri Mar 7 17:14:17 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Fri Mar 7 17:14:17 2014 -0500

----------------------------------------------------------------------
 .../apache/accumulo/server/client/BulkImporter.java |  4 ++--
 .../accumulo/server/client/HdfsZooInstance.java     | 15 ++++++++++++++-
 .../accumulo/server/conf/ZooConfiguration.java      | 16 +++++++++++++++-
 .../org/apache/accumulo/server/fs/VolumeUtil.java   | 10 ++--------
 .../org/apache/accumulo/server/init/Initialize.java |  2 +-
 .../server/master/recovery/HadoopLogCloser.java     |  2 +-
 .../server/master/recovery/MapRLogCloser.java       |  2 +-
 .../apache/accumulo/server/util/LocalityCheck.java  |  2 +-
 .../accumulo/server/util/TabletOperations.java      |  4 ++--
 9 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index dc9acf8..27ab078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -339,7 +339,7 @@ public class BulkImporter {
     
     try {
       for (Path path : paths) {
-        FileSystem fs = vm.getFileSystemByPath(path);
+        FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
         mapFileSizes.put(path, fs.getContentSummary(path).getLength());
       }
     } catch (IOException e) {
@@ -639,7 +639,7 @@ public class BulkImporter {
     Collection<ByteSequence> columnFamilies = Collections.emptyList();
     String filename = file.toString();
     // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
-    FileSystem fs = vm.getFileSystemByPath(file);
+    FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
     FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs,
fs.getConf(), acuConf);
     try {
       Text row = startRow;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6993a0a..8aa0be9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.server.client;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.List;
@@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
 import org.apache.accumulo.core.util.OpTimer;
 import org.apache.accumulo.core.util.StringUtil;
 import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.volume.Volume;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.fate.zookeeper.ZooCache;
 import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
 import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Level;
@@ -122,7 +126,16 @@ public class HdfsZooInstance implements Instance {
 
   private static synchronized void _getInstanceID() {
     if (instanceId == null) {
-      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(),
ServerConfiguration.getSiteConfiguration());
+      AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+      // InstanceID should be the same across all volumes, so just choose one
+      VolumeManager fs;
+      try {
+        fs = VolumeManagerImpl.get();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+      Volume randVolume = fs.getVolumes().iterator().next();
+      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume),
acuConf);
       instanceId = instanceIdFromFile;
     }
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 32f6126..94e468b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.server.conf;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.fate.zookeeper.ZooCache;
 import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
 import org.apache.log4j.Logger;
 
 public class ZooConfiguration extends AccumuloConfiguration {
@@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
     if (instance == null) {
       propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
       instance = new ZooConfiguration(parent);
-      String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(),
parent);
+      // InstanceID should be the same across all volumes, so just choose one
+      VolumeManager fs;
+      try {
+        fs = VolumeManagerImpl.get();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+      Volume randVolume = fs.getVolumes().iterator().next();
+      String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume),
parent);
       instanceId = deprecatedInstanceIdFromHdfs;
     }
     return instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 42ea912..110f1e4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -26,21 +26,17 @@ import java.util.Map.Entry;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.tabletserver.log.LogEntry;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.volume.Volume;
 import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.fs.VolumeManager.FileType;
 import org.apache.accumulo.server.security.SystemCredentials;
 import org.apache.accumulo.server.util.MetadataTableUtil;
 import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -49,8 +45,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 
-import com.google.common.base.Preconditions;
-
 /**
  * Utility methods for managing absolute URIs contained in Accumulo metadata.
  */
@@ -245,8 +239,8 @@ public class VolumeUtil {
 
       // this code needs to be idempotent
 
-      FileSystem fs1 = vm.getFileSystemByPath(dir);
-      FileSystem fs2 = vm.getFileSystemByPath(newDir);
+      FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
+      FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();
 
       if (!same(fs1, dir, fs2, newDir)) {
         if (fs2.exists(newDir)) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 59fad73..206a721 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -341,7 +341,7 @@ public class Initialize {
     // the root tablet contains the key extent and locations of all the
     // metadata tablets
     String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
-    FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
+    FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem();
     FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(),
AccumuloConfiguration.getDefaultConfiguration());
     mfw.startDefaultLocalityGroup();
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 7edc0cf..0e7f83a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser {
 
   @Override
   public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException
{
-    FileSystem ns = fs.getFileSystemByPath(source);
+    FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
 
     // if path points to a viewfs path, then resolve to underlying filesystem
     if (ViewFSUtils.isViewFS(ns)) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
index bba7ac5..d737109 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
@@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser {
   @Override
   public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException
{
     log.info("Recovering file " + path.toString() + " by changing permission to readonly");
-    FileSystem ns = fs.getFileSystemByPath(path);
+    FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
     FsPermission roPerm = new FsPermission((short) 0444);
     try {
       ns.setPermission(path, roPerm);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index a96e791..35bb8f5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -88,7 +88,7 @@ public class LocalityCheck {
     }
     for (String file : files) {
       Path filePath = new Path(file);
-      FileSystem ns = fs.getFileSystemByPath(filePath);
+      FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
       FileStatus fileStatus = ns.getFileStatus(filePath);
       BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
       for (BlockLocation blockLocation : fileBlockLocations) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
index b237cd0..b8e7113 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
@@ -46,7 +46,7 @@ public class TabletOperations {
           lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
           Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
           if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
-            FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath);
+            FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
             return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString();
           }
           log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
@@ -56,7 +56,7 @@ public class TabletOperations {
           if (fs.exists(lowDirectoryPath))
             throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
           if (fs.mkdirs(lowDirectoryPath)) {
-            FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath);
+            FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
             return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString();
           }
         }


Mime
View raw message