accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [3/5] git commit: ACCUMULO-1764 Iterate over each volume and compute an overall summary of usage.
Date Fri, 14 Feb 2014 02:18:28 GMT
ACCUMULO-1764 Iterate over each volume and compute an overall summary of usage.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75aac15d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75aac15d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75aac15d

Branch: refs/heads/master
Commit: 75aac15d568d5a2d0a50b9bbe813551615d14d1f
Parents: ec4d4d8
Author: Josh Elser <elserj@apache.org>
Authored: Thu Feb 13 19:25:11 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Feb 13 19:26:56 2014 -0500

----------------------------------------------------------------------
 .../monitor/servlets/DefaultServlet.java        | 70 ++++++++++++++------
 1 file changed, 51 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/75aac15d/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 4a7838a..75aa400 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@ -33,10 +33,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.file.FileUtil;
 import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
-import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.Duration;
 import org.apache.accumulo.core.util.NumUtil;
 import org.apache.accumulo.core.util.Pair;
@@ -44,9 +41,10 @@ import org.apache.accumulo.monitor.Monitor;
 import org.apache.accumulo.monitor.ZooKeeperStatus;
 import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
 import org.apache.accumulo.monitor.util.celltypes.NumberType;
+import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.trace.TraceFileSystem;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -259,31 +257,65 @@ public class DefaultServlet extends BasicServlet {
 
   private void doAccumuloTable(StringBuilder sb) throws IOException {
     // Accumulo
-    Configuration conf = CachedConfiguration.getInstance();
-    FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+    VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
     MasterMonitorInfo info = Monitor.getMmi();
     sb.append("<table>\n");
     sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
     if (info == null) {
       sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
     } else {
-      String consumed = "Unknown";
-      String diskUsed = "Unknown";
+      long totalAcuBytesUsed = 0l;
+      long totalHdfsBytesUsed = 0l;
+      
       try {
-        Path path = new Path(Monitor.getSystemConfiguration().get(Property.INSTANCE_DFS_DIR));
-        log.debug("Reading the content summary for " + path);
-        try {
-          ContentSummary acu = fs.getContentSummary(path);
-          diskUsed = bytes(acu.getSpaceConsumed());
-          ContentSummary rootSummary = fs.getContentSummary(new Path("/"));
-          consumed = String.format("%.2f%%", acu.getSpaceConsumed() * 100. / rootSummary.getSpaceConsumed());
-        } catch (Exception ex) {
-          log.trace("Unable to get disk usage information from hdfs", ex);
+        for (String baseDir : ServerConstants.getConfiguredBaseDirs()) {
+          final Path basePath = new Path(baseDir);
+          final FileSystem fs = vm.getFileSystemByPath(basePath);
+          
+          try {
+            // Calculate the amount of space used by Accumulo on the FileSystem
+            ContentSummary accumuloSummary = fs.getContentSummary(basePath);
+            long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
+            totalAcuBytesUsed += bytesUsedByAcuOnFs;
+
+            // Catch the overflow -- this is big data
+            if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
+              log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
+              totalAcuBytesUsed = 0l;
+              break;
+            }
+
+            // Calculate the total amount of space used on the FileSystem
+            ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
+            long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
+            totalHdfsBytesUsed += bytesUsedOnVolume;
+
+            // Catch the overflow -- this is big data
+            if (totalHdfsBytesUsed < bytesUsedOnVolume) {
+              log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
+              totalHdfsBytesUsed = 0;
+              break;
+            }
+          } catch (Exception ex) {
+            log.trace("Unable to get disk usage information for " + baseDir, ex);
+          }
+        }
+
+        String diskUsed = "Unknown";
+        String consumed = null;
+        if (totalAcuBytesUsed > 0) {
+          // Convert Accumulo usage to a readable String
+          diskUsed = bytes(totalAcuBytesUsed);
+          
+          if (totalHdfsBytesUsed > 0) {
+            // Compute amount of space used by Accumulo as a percentage of total space usage.
+            consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
+          }
         }
 
         boolean highlight = false;
         tableRow(sb, (highlight = !highlight), "Disk&nbsp;Used", diskUsed);
-        if (fs.getUsed() != 0)
+        if (null != consumed)
           tableRow(sb, (highlight = !highlight), "%&nbsp;of&nbsp;Used&nbsp;DFS",
consumed);
         tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>",
NumberType.commas(Monitor.getTotalTables()));
         tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet&nbsp;Servers</a>",
NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));


Mime
View raw message