accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject accumulo git commit: ACCUMULO-3238 Refactored TableDiskUsage
Date Fri, 18 Aug 2017 19:29:53 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master cc1cf734b -> b6a211145


ACCUMULO-3238 Refactored TableDiskUsage


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b6a21114
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b6a21114
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b6a21114

Branch: refs/heads/master
Commit: b6a211145378ed0f903b1bdf1980beb1238da12d
Parents: cc1cf73
Author: Mike Miller <mmiller@apache.org>
Authored: Fri Aug 18 15:23:39 2017 -0400
Committer: Mike Miller <mmiller@apache.org>
Committed: Fri Aug 18 15:28:51 2017 -0400

----------------------------------------------------------------------
 .../server/client/ClientServiceHandler.java     |   7 +-
 .../accumulo/server/util/TableDiskUsage.java    | 111 ++++++++-----------
 2 files changed, 51 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/b6a21114/server/base/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
b/server/base/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
index 343cfc4..abfb160 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
@@ -431,20 +431,19 @@ public class ClientServiceHandler implements ClientService.Iface {
   @Override
   public List<TDiskUsage> getDiskUsage(Set<String> tables, TCredentials credentials)
throws ThriftTableOperationException, ThriftSecurityException, TException {
     try {
-      HashSet<String> tableIds = new HashSet<>();
+      HashSet<Table.ID> tableIds = new HashSet<>();
 
       for (String table : tables) {
         // ensure that table table exists
         Table.ID tableId = checkTableId(instance, table, null);
-        tableIds.add(tableId.canonicalID());
+        tableIds.add(tableId);
         Namespace.ID namespaceId = Tables.getNamespaceId(instance, tableId);
         if (!security.canScan(credentials, tableId, namespaceId))
           throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
       }
 
       // use the same set of tableIds that were validated above to avoid race conditions
-      Map<TreeSet<String>,Long> diskUsage = TableDiskUsage.getDiskUsage(context.getServerConfigurationFactory().getSystemConfiguration(),
tableIds, fs,
-          context.getConnector());
+      Map<TreeSet<String>,Long> diskUsage = TableDiskUsage.getDiskUsage(tableIds,
fs, context.getConnector());
       List<TDiskUsage> retUsages = new ArrayList<>();
       for (Map.Entry<TreeSet<String>,Long> usageItem : diskUsage.entrySet())
{
         retUsages.add(new TDiskUsage(new ArrayList<>(usageItem.getKey()), usageItem.getValue()));

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b6a21114/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index e1e52ae..f36de51 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -20,7 +20,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -35,8 +34,7 @@ import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.Table;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.data.impl.KeyExtent;
@@ -59,12 +57,12 @@ public class TableDiskUsage {
 
   private static final Logger log = LoggerFactory.getLogger(TableDiskUsage.class);
   private int nextInternalId = 0;
-  private Map<String,Integer> internalIds = new HashMap<>();
-  private Map<Integer,String> externalIds = new HashMap<>();
+  private Map<Table.ID,Integer> internalIds = new HashMap<>();
+  private Map<Integer,Table.ID> externalIds = new HashMap<>();
   private Map<String,Integer[]> tableFiles = new HashMap<>();
   private Map<String,Long> fileSizes = new HashMap<>();
 
-  void addTable(String tableId) {
+  void addTable(Table.ID tableId) {
     if (internalIds.containsKey(tableId))
       throw new IllegalArgumentException("Already added table " + tableId);
 
@@ -77,7 +75,7 @@ public class TableDiskUsage {
     externalIds.put(iid, tableId);
   }
 
-  void linkFileAndTable(String tableId, String file) {
+  void linkFileAndTable(Table.ID tableId, String file) {
     // get the internal id for this table
     int internalId = internalIds.get(tableId);
 
@@ -98,7 +96,7 @@ public class TableDiskUsage {
     fileSizes.put(file, size);
   }
 
-  Map<List<String>,Long> calculateUsage() {
+  Map<List<Table.ID>,Long> calculateUsage() {
 
     // Bitset of tables that contain a file and total usage by all files that share that
usage
     Map<List<Integer>,Long> usage = new HashMap<>();
@@ -124,10 +122,10 @@ public class TableDiskUsage {
 
     }
 
-    Map<List<String>,Long> externalUsage = new HashMap<>();
+    Map<List<Table.ID>,Long> externalUsage = new HashMap<>();
 
     for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
-      List<String> externalKey = new ArrayList<>();
+      List<Table.ID> externalKey = new ArrayList<>();
       List<Integer> key = entry.getKey();
       // table bitset
       for (int i = 0; i < key.size(); i++)
@@ -147,38 +145,32 @@ public class TableDiskUsage {
     void print(String line);
   }
 
-  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String>
tables, VolumeManager fs, Connector conn, boolean humanReadable)
-      throws TableNotFoundException, IOException {
-    printDiskUsage(acuConf, tables, fs, conn, new Printer() {
-      @Override
-      public void print(String line) {
-        System.out.println(line);
-      }
-    }, humanReadable);
+  public static void printDiskUsage(Collection<String> tableNames, VolumeManager fs,
Connector conn, boolean humanReadable) throws TableNotFoundException,
+      IOException {
+    printDiskUsage(tableNames, fs, conn, line -> System.out.println(line), humanReadable);
   }
 
-  public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration
acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
-      throws IOException {
+  public static Map<TreeSet<String>,Long> getDiskUsage(Set<Table.ID> tableIds,
VolumeManager fs, Connector conn) throws IOException {
     TableDiskUsage tdu = new TableDiskUsage();
 
     // Add each tableID
-    for (String tableId : tableIds)
+    for (Table.ID tableId : tableIds)
       tdu.addTable(tableId);
 
-    HashSet<String> tablesReferenced = new HashSet<>(tableIds);
-    HashSet<String> emptyTableIds = new HashSet<>();
+    HashSet<Table.ID> tablesReferenced = new HashSet<>(tableIds);
+    HashSet<Table.ID> emptyTableIds = new HashSet<>();
     HashSet<String> nameSpacesReferenced = new HashSet<>();
 
     // For each table ID
-    for (String tableId : tableIds) {
-      Scanner mdScanner = null;
+    for (Table.ID tableId : tableIds) {
+      Scanner mdScanner;
       try {
         mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
       } catch (TableNotFoundException e) {
         throw new RuntimeException(e);
       }
       mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
-      mdScanner.setRange(new KeyExtent(Table.ID.of(tableId), null, null).toMetadataRange());
+      mdScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
 
       if (!mdScanner.iterator().hasNext()) {
         emptyTableIds.add(tableId);
@@ -193,8 +185,8 @@ public class TableDiskUsage {
         if (file.contains(":") || file.startsWith("../")) {
           String ref = parts[parts.length - 3];
           // Track any tables which are referenced externally by the current table
-          if (!ref.equals(tableId)) {
-            tablesReferenced.add(ref);
+          if (!ref.equals(tableId.canonicalID())) {
+            tablesReferenced.add(Table.ID.of(ref));
           }
           if (file.contains(":") && parts.length > 3) {
             List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
@@ -208,7 +200,7 @@ public class TableDiskUsage {
     }
 
     // Each table seen (provided by user, or reference by table the user provided)
-    for (String tableId : tablesReferenced) {
+    for (Table.ID tableId : tablesReferenced) {
       for (String tableDir : nameSpacesReferenced) {
         // Find each file and add its size
         FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
@@ -222,45 +214,38 @@ public class TableDiskUsage {
       }
     }
 
-    // Invert tableId->tableName
-    HashMap<String,String> reverseTableIdMap = new HashMap<>();
-    for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
-      reverseTableIdMap.put(entry.getValue(), entry.getKey());
+    Map<Table.ID,String> reverseTableIdMap = Tables.getIdToNameMap(conn.getInstance());
 
-    TreeMap<TreeSet<String>,Long> usage = new TreeMap<>(new Comparator<TreeSet<String>>()
{
+    TreeMap<TreeSet<String>,Long> usage = new TreeMap<>((o1, o2) ->
{
+      int len1 = o1.size();
+      int len2 = o2.size();
 
-      @Override
-      public int compare(TreeSet<String> o1, TreeSet<String> o2) {
-        int len1 = o1.size();
-        int len2 = o2.size();
+      int min = Math.min(len1, len2);
 
-        int min = Math.min(len1, len2);
+      Iterator<String> iter1 = o1.iterator();
+      Iterator<String> iter2 = o2.iterator();
 
-        Iterator<String> iter1 = o1.iterator();
-        Iterator<String> iter2 = o2.iterator();
+      int count = 0;
 
-        int count = 0;
+      while (count < min) {
+        String s1 = iter1.next();
+        String s2 = iter2.next();
 
-        while (count < min) {
-          String s1 = iter1.next();
-          String s2 = iter2.next();
+        int cmp = s1.compareTo(s2);
 
-          int cmp = s1.compareTo(s2);
+        if (cmp != 0)
+          return cmp;
 
-          if (cmp != 0)
-            return cmp;
-
-          count++;
-        }
-
-        return len1 - len2;
+        count++;
       }
+
+      return len1 - len2;
     });
 
-    for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+    for (Entry<List<Table.ID>,Long> entry : tdu.calculateUsage().entrySet())
{
       TreeSet<String> tableNames = new TreeSet<>();
       // Convert size shared by each table id into size shared by each table name
-      for (String tableId : entry.getKey())
+      for (Table.ID tableId : entry.getKey())
         tableNames.add(reverseTableIdMap.get(tableId));
 
       // Make table names to shared file size
@@ -269,7 +254,7 @@ public class TableDiskUsage {
 
     if (!emptyTableIds.isEmpty()) {
       TreeSet<String> emptyTables = new TreeSet<>();
-      for (String tableId : emptyTableIds) {
+      for (Table.ID tableId : emptyTableIds) {
         emptyTables.add(reverseTableIdMap.get(tableId));
       }
       usage.put(emptyTables, 0L);
@@ -278,21 +263,21 @@ public class TableDiskUsage {
     return usage;
   }
 
-  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String>
tables, VolumeManager fs, Connector conn, Printer printer,
-      boolean humanReadable) throws TableNotFoundException, IOException {
+  public static void printDiskUsage(Collection<String> tableNames, VolumeManager fs,
Connector conn, Printer printer, boolean humanReadable)
+      throws TableNotFoundException, IOException {
 
-    HashSet<String> tableIds = new HashSet<>();
+    HashSet<Table.ID> tableIds = new HashSet<>();
 
     // Get table IDs for all tables requested to be 'du'
-    for (String tableName : tables) {
-      String tableId = conn.tableOperations().tableIdMap().get(tableName);
+    for (String tableName : tableNames) {
+      Table.ID tableId = Tables.getTableId(conn.getInstance(), tableName);
       if (tableId == null)
         throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
 
       tableIds.add(tableId);
     }
 
-    Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+    Map<TreeSet<String>,Long> usage = getDiskUsage(tableIds, fs, conn);
 
     String valueFormat = humanReadable ? "%9s" : "%,24d";
     for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
@@ -311,7 +296,7 @@ public class TableDiskUsage {
     Opts opts = new Opts();
     opts.parseArgs(TableDiskUsage.class.getName(), args);
     Connector conn = opts.getConnector();
-    org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(),
opts.tables, fs, conn, false);
+    org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(opts.tables, fs, conn,
false);
   }
 
 }


Mime
View raw message