Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C2FFE17520 for ; Thu, 12 Feb 2015 22:52:54 +0000 (UTC) Received: (qmail 71491 invoked by uid 500); 12 Feb 2015 22:52:54 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 71378 invoked by uid 500); 12 Feb 2015 22:52:54 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 71279 invoked by uid 99); 12 Feb 2015 22:52:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Feb 2015 22:52:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6D9CFE03EB; Thu, 12 Feb 2015 22:52:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Thu, 12 Feb 2015 22:52:55 -0000 Message-Id: <8afb9fa2062f436fae552e8abdf6fbc8@git.apache.org> In-Reply-To: <82abc75723b247ee972e04fe93d0b83d@git.apache.org> References: <82abc75723b247ee972e04fe93d0b83d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/5] accumulo git commit: ACCUMULO-3585 Add comments to TableDiskUsage. ACCUMULO-3585 Add comments to TableDiskUsage. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/653895f3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/653895f3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/653895f3 Branch: refs/heads/master Commit: 653895f374502342c674bdf6db4f76bcc9ccef01 Parents: 1297dc7 Author: Josh Elser Authored: Thu Feb 12 17:31:42 2015 -0500 Committer: Josh Elser Committed: Thu Feb 12 17:31:42 2015 -0500 ---------------------------------------------------------------------- .../accumulo/server/util/TableDiskUsage.java | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/653895f3/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java index 24eac05..e48e10a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java @@ -67,15 +67,20 @@ public class TableDiskUsage { if (internalIds.containsKey(tableId)) throw new IllegalArgumentException("Already added table " + tableId); + // Keep an internal counter for each table added int iid = nextInternalId++; + // Store the table id to the internal id internalIds.put(tableId, iid); + // Store the internal id to the table id externalIds.put(iid, tableId); } void linkFileAndTable(String tableId, String file) { + // get the internal id for this table int internalId = internalIds.get(tableId); + // Initialize a bitset for tables (internal IDs) that reference this file Integer[] tables = tableFiles.get(file); if (tables == null) { tables = new Integer[internalIds.size()]; @@ -84,6 +89,7 @@ public class TableDiskUsage { tableFiles.put(file, tables); } + // Update the bitset to track that this table has seen this file tables[internalId] = 1; } @@ -93,8 +99,10 @@ public class TableDiskUsage { Map,Long> calculateUsage() { + // Bitset of tables that contain a file and total usage by all files that share that usage Map,Long> usage = new HashMap,Long>(); + // For each file w/ referenced-table bitset for (Entry entry : tableFiles.entrySet()) { if (log.isTraceEnabled()) { log.trace("fileSizes " + fileSizes + " key " + entry.getKey()); @@ -117,13 +125,17 @@ public class TableDiskUsage { for (Entry,Long> entry : usage.entrySet()) { List externalKey = new ArrayList(); List key = entry.getKey(); + // table bitset for (int i = 0; i < key.size(); i++) if (key.get(i) != 0) + // Convert by internal id to the table id externalKey.add(externalIds.get(i)); + // list of table ids and size of files shared across the tables externalUsage.put(externalKey, entry.getValue()); } + // mapping of all enumerations of files being referenced by tables and total size of files who share the same reference return externalUsage; } @@ -145,6 +157,7 @@ public class TableDiskUsage { throws IOException { TableDiskUsage tdu = new TableDiskUsage(); + // Add each tableID for (String tableId : tableIds) tdu.addTable(tableId); @@ -152,6 +165,7 @@ public class TableDiskUsage { HashSet emptyTableIds = new HashSet(); HashSet nameSpacesReferenced = new HashSet(); + // For each table ID for (String tableId : tableIds) { Scanner mdScanner = null; try { @@ -166,12 +180,15 @@ public class TableDiskUsage { emptyTableIds.add(tableId); } + // Read each file referenced by that table for (Entry entry : mdScanner) { String file = entry.getKey().getColumnQualifier().toString(); String parts[] = file.split("/"); + // the filename String uniqueName = parts[parts.length - 1]; if (file.contains(":") || file.startsWith("../")) { String ref = parts[parts.length - 3]; + // Track any tables which are referenced externally by the current table if (!ref.equals(tableId)) { tablesReferenced.add(ref); } @@ -181,12 +198,15 @@ public class TableDiskUsage { } } + // add this file to this table tdu.linkFileAndTable(tableId, uniqueName); } } + // Each table seen (provided by user, or reference by table the user provided) for (String tableId : tablesReferenced) { for (String tableDir : nameSpacesReferenced) { + // Find each file and add its size FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*")); if (files != null) { for (FileStatus fileStatus : files) { @@ -198,6 +218,7 @@ public class TableDiskUsage { } } + // Invert tableId->tableName HashMap reverseTableIdMap = new HashMap(); for (Entry entry : conn.tableOperations().tableIdMap().entrySet()) reverseTableIdMap.put(entry.getValue(), entry.getKey()); @@ -234,9 +255,11 @@ public class TableDiskUsage { for (Entry,Long> entry : tdu.calculateUsage().entrySet()) { TreeSet tableNames = new TreeSet(); + // Convert size shared by each table id into size shared by each table name for (String tableId : entry.getKey()) tableNames.add(reverseTableIdMap.get(tableId)); + // Make table names to shared file size usage.put(tableNames, entry.getValue()); } @@ -256,6 +279,7 @@ public class TableDiskUsage { HashSet tableIds = new HashSet(); + // Get table IDs for all tables requested to be 'du' for (String tableName : tables) { String tableId = conn.tableOperations().tableIdMap().get(tableName); if (tableId == null)