Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F26AF10FCE for ; Tue, 11 Mar 2014 18:25:36 +0000 (UTC) Received: (qmail 77296 invoked by uid 500); 11 Mar 2014 18:25:16 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 77119 invoked by uid 500); 11 Mar 2014 18:25:13 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 76996 invoked by uid 99); 11 Mar 2014 18:25:10 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 11 Mar 2014 18:25:10 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id AF51D94060D; Tue, 11 Mar 2014 18:25:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Tue, 11 Mar 2014 18:25:13 -0000 Message-Id: In-Reply-To: <7d657461298f48d5a941df4e09887beb@git.apache.org> References: <7d657461298f48d5a941df4e09887beb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/23] git commit: ACCUMULO-2061 Fix the rest of server-base ACCUMULO-2061 Fix the rest of server-base Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e613f710 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e613f710 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e613f710 Branch: refs/heads/ACCUMULO-2061 Commit: e613f7105e3a0828096909535147b4bc80a265f6 Parents: 3b3ccbb Author: Josh Elser Authored: Fri Mar 7 17:14:17 2014 -0500 Committer: Josh Elser Committed: Fri Mar 7 17:14:17 2014 -0500 ---------------------------------------------------------------------- .../apache/accumulo/server/client/BulkImporter.java | 4 ++-- .../accumulo/server/client/HdfsZooInstance.java | 15 ++++++++++++++- .../accumulo/server/conf/ZooConfiguration.java | 16 +++++++++++++++- .../org/apache/accumulo/server/fs/VolumeUtil.java | 10 ++-------- .../org/apache/accumulo/server/init/Initialize.java | 2 +- .../server/master/recovery/HadoopLogCloser.java | 2 +- .../server/master/recovery/MapRLogCloser.java | 2 +- .../apache/accumulo/server/util/LocalityCheck.java | 2 +- .../accumulo/server/util/TabletOperations.java | 4 ++-- 9 files changed, 39 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java index dc9acf8..27ab078 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java +++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java @@ -339,7 +339,7 @@ public class BulkImporter { try { for (Path path : paths) { - FileSystem fs = vm.getFileSystemByPath(path); + FileSystem fs = vm.getVolumeByPath(path).getFileSystem(); mapFileSizes.put(path, fs.getContentSummary(path).getLength()); } } catch (IOException e) { @@ -639,7 +639,7 @@ public class BulkImporter { Collection columnFamilies = Collections.emptyList(); String filename = file.toString(); // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow); - FileSystem fs = vm.getFileSystemByPath(file); + FileSystem fs = vm.getVolumeByPath(file).getFileSystem(); FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf); try { Text row = startRow; http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java index 6993a0a..8aa0be9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java +++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java @@ -16,6 +16,7 @@ */ package org.apache.accumulo.server.client; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.OpTimer; import org.apache.accumulo.core.util.StringUtil; import org.apache.accumulo.core.util.TextUtil; +import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.ZooCache; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.zookeeper.ZooLock; import org.apache.hadoop.io.Text; import org.apache.log4j.Level; @@ -122,7 +126,16 @@ public class HdfsZooInstance implements Instance { private static synchronized void _getInstanceID() { if (instanceId == null) { - String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), ServerConfiguration.getSiteConfiguration()); + AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration(); + // InstanceID should be the same across all volumes, so just choose one + VolumeManager fs; + try { + fs = VolumeManagerImpl.get(); + } catch (IOException e) { + throw new RuntimeException(e); + } + Volume randVolume = fs.getVolumes().iterator().next(); + String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf); instanceId = instanceIdFromFile; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java index 32f6126..94e468b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java @@ -16,6 +16,7 @@ */ package org.apache.accumulo.server.conf; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.volume.Volume; +import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.ZooCache; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.log4j.Logger; public class ZooConfiguration extends AccumuloConfiguration { @@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration { if (instance == null) { propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)); instance = new ZooConfiguration(parent); - String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), parent); + // InstanceID should be the same across all volumes, so just choose one + VolumeManager fs; + try { + fs = VolumeManagerImpl.get(); + } catch (IOException e) { + throw new RuntimeException(e); + } + Volume randVolume = fs.getVolumes().iterator().next(); + String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent); instanceId = deprecatedInstanceIdFromHdfs; } return instance; http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index 42ea912..110f1e4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -26,21 +26,17 @@ import java.util.Map.Entry; import java.util.SortedMap; import java.util.TreeMap; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.Pair; -import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.accumulo.server.zookeeper.ZooLock; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -49,8 +45,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import com.google.common.base.Preconditions; - /** * Utility methods for managing absolute URIs contained in Accumulo metadata. */ @@ -245,8 +239,8 @@ public class VolumeUtil { // this code needs to be idempotent - FileSystem fs1 = vm.getFileSystemByPath(dir); - FileSystem fs2 = vm.getFileSystemByPath(newDir); + FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem(); + FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem(); if (!same(fs1, dir, fs2, newDir)) { if (fs2.exists(newDir)) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index 59fad73..206a721 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@ -341,7 +341,7 @@ public class Initialize { // the root tablet contains the key extent and locations of all the // metadata tablets String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration()); - FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile)); + FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem(); FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration()); mfw.startDefaultLocalityGroup(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java index 7edc0cf..0e7f83a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java @@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser { @Override public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException { - FileSystem ns = fs.getFileSystemByPath(source); + FileSystem ns = fs.getVolumeByPath(source).getFileSystem(); // if path points to a viewfs path, then resolve to underlying filesystem if (ViewFSUtils.isViewFS(ns)) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java index bba7ac5..d737109 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java @@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser { @Override public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException { log.info("Recovering file " + path.toString() + " by changing permission to readonly"); - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); FsPermission roPerm = new FsPermission((short) 0444); try { ns.setPermission(path, roPerm); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java index a96e791..35bb8f5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java @@ -88,7 +88,7 @@ public class LocalityCheck { } for (String file : files) { Path filePath = new Path(file); - FileSystem ns = fs.getFileSystemByPath(filePath); + FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem(); FileStatus fileStatus = ns.getFileStatus(filePath); BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java index b237cd0..b8e7113 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java @@ -46,7 +46,7 @@ public class TabletOperations { lowDirectory = Constants.DEFAULT_TABLET_LOCATION; Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory); if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) { - FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath); + FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem(); return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString(); } log.warn("Failed to create " + lowDirectoryPath + " for unknown reason"); @@ -56,7 +56,7 @@ public class TabletOperations { if (fs.exists(lowDirectoryPath)) throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath); if (fs.mkdirs(lowDirectoryPath)) { - FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath); + FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem(); return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString(); } }