accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [1/5] accumulo git commit: ACCUMULO-3585 Add comments to TableDiskUsage.
Date Thu, 12 Feb 2015 22:52:54 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 1297dc7bf -> 6b12b4746
  refs/heads/master 8a34fa875 -> b620567e3


ACCUMULO-3585 Add comments to TableDiskUsage.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/653895f3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/653895f3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/653895f3

Branch: refs/heads/1.6
Commit: 653895f374502342c674bdf6db4f76bcc9ccef01
Parents: 1297dc7
Author: Josh Elser <elserj@apache.org>
Authored: Thu Feb 12 17:31:42 2015 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Feb 12 17:31:42 2015 -0500

----------------------------------------------------------------------
 .../accumulo/server/util/TableDiskUsage.java    | 24 ++++++++++++++++++++
 1 file changed, 24 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/653895f3/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index 24eac05..e48e10a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -67,15 +67,20 @@ public class TableDiskUsage {
     if (internalIds.containsKey(tableId))
       throw new IllegalArgumentException("Already added table " + tableId);
 
+    // Keep an internal counter for each table added
     int iid = nextInternalId++;
 
+    // Store the table id to the internal id
     internalIds.put(tableId, iid);
+    // Store the internal id to the table id
     externalIds.put(iid, tableId);
   }
 
   void linkFileAndTable(String tableId, String file) {
+    // get the internal id for this table
     int internalId = internalIds.get(tableId);
 
+    // Initialize a bitset for tables (internal IDs) that reference this file
     Integer[] tables = tableFiles.get(file);
     if (tables == null) {
       tables = new Integer[internalIds.size()];
@@ -84,6 +89,7 @@ public class TableDiskUsage {
       tableFiles.put(file, tables);
     }
 
+    // Update the bitset to track that this table has seen this file
     tables[internalId] = 1;
   }
 
@@ -93,8 +99,10 @@ public class TableDiskUsage {
 
   Map<List<String>,Long> calculateUsage() {
 
+    // Bitset of tables that contain a file and total usage by all files that share that
usage
     Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
 
+    // For each file w/ referenced-table bitset
     for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
       if (log.isTraceEnabled()) {
         log.trace("fileSizes " + fileSizes + " key " + entry.getKey());
@@ -117,13 +125,17 @@ public class TableDiskUsage {
     for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
       List<String> externalKey = new ArrayList<String>();
       List<Integer> key = entry.getKey();
+      // table bitset
       for (int i = 0; i < key.size(); i++)
         if (key.get(i) != 0)
+          // Convert by internal id to the table id
           externalKey.add(externalIds.get(i));
 
+      // list of table ids and size of files shared across the tables
       externalUsage.put(externalKey, entry.getValue());
     }
 
+    // mapping of all enumerations of files being referenced by tables and total size of
files who share the same reference
     return externalUsage;
   }
 
@@ -145,6 +157,7 @@ public class TableDiskUsage {
       throws IOException {
     TableDiskUsage tdu = new TableDiskUsage();
 
+    // Add each tableID
     for (String tableId : tableIds)
       tdu.addTable(tableId);
 
@@ -152,6 +165,7 @@ public class TableDiskUsage {
     HashSet<String> emptyTableIds = new HashSet<String>();
     HashSet<String> nameSpacesReferenced = new HashSet<String>();
 
+    // For each table ID
     for (String tableId : tableIds) {
       Scanner mdScanner = null;
       try {
@@ -166,12 +180,15 @@ public class TableDiskUsage {
         emptyTableIds.add(tableId);
       }
 
+      // Read each file referenced by that table
       for (Entry<Key,Value> entry : mdScanner) {
         String file = entry.getKey().getColumnQualifier().toString();
         String parts[] = file.split("/");
+        // the filename
         String uniqueName = parts[parts.length - 1];
         if (file.contains(":") || file.startsWith("../")) {
           String ref = parts[parts.length - 3];
+          // Track any tables which are referenced externally by the current table
           if (!ref.equals(tableId)) {
             tablesReferenced.add(ref);
           }
@@ -181,12 +198,15 @@ public class TableDiskUsage {
           }
         }
 
+        // add this file to this table
         tdu.linkFileAndTable(tableId, uniqueName);
       }
     }
 
+    // Each table seen (provided by user, or reference by table the user provided)
     for (String tableId : tablesReferenced) {
       for (String tableDir : nameSpacesReferenced) {
+        // Find each file and add its size
         FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
         if (files != null) {
           for (FileStatus fileStatus : files) {
@@ -198,6 +218,7 @@ public class TableDiskUsage {
       }
     }
 
+    // Invert tableId->tableName
     HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
     for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
       reverseTableIdMap.put(entry.getValue(), entry.getKey());
@@ -234,9 +255,11 @@ public class TableDiskUsage {
 
     for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
       TreeSet<String> tableNames = new TreeSet<String>();
+      // Convert size shared by each table id into size shared by each table name
       for (String tableId : entry.getKey())
         tableNames.add(reverseTableIdMap.get(tableId));
 
+      // Make table names to shared file size
       usage.put(tableNames, entry.getValue());
     }
 
@@ -256,6 +279,7 @@ public class TableDiskUsage {
 
     HashSet<String> tableIds = new HashSet<String>();
 
+    // Get table IDs for all tables requested to be 'du'
     for (String tableName : tables) {
       String tableId = conn.tableOperations().tableIdMap().get(tableName);
       if (tableId == null)


Mime
View raw message