accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1493756 [2/2] - in /accumulo/branches/ACCUMULO-118: core/src/main/java/org/apache/accumulo/core/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apac...
Date Mon, 17 Jun 2013 13:26:43 GMT
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
Mon Jun 17 13:26:43 2013
@@ -66,7 +66,7 @@ public class RemoveEntriesForMissingFile
     for (Entry<Key,Value> entry : metadata) {
       count++;
       Key key = entry.getKey();
-      Path map = new Path(fs.getFullPath(key));
+      Path map = fs.getFullPath(key);
       if (!fs.exists(map)) {
         missing++;
         log.info("File " + map + " is missing");

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
Mon Jun 17 13:26:43 2013
@@ -16,24 +16,262 @@
  */
 package org.apache.accumulo.server.util;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
 
-import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
 
 import com.beust.jcommander.Parameter;
 
 public class TableDiskUsage {
   
+  
+  private static final Logger log = Logger.getLogger(Logger.class);
+  private int nextInternalId = 0;
+  private Map<String,Integer> internalIds = new HashMap<String,Integer>();
+  private Map<Integer,String> externalIds = new HashMap<Integer,String>();
+  private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
+  private Map<String,Long> fileSizes = new HashMap<String,Long>();
+  
+  void addTable(String tableId) {
+    if (internalIds.containsKey(tableId))
+      throw new IllegalArgumentException("Already added table " + tableId);
+    
+    int iid = nextInternalId++;
+    
+    internalIds.put(tableId, iid);
+    externalIds.put(iid, tableId);
+  }
+  
+  void linkFileAndTable(String tableId, String file) {
+    int internalId = internalIds.get(tableId);
+    
+    Integer[] tables = tableFiles.get(file);
+    if (tables == null) {
+      tables = new Integer[internalIds.size()];
+      for (int i = 0; i < tables.length; i++)
+        tables[i] = 0;
+      tableFiles.put(file, tables);
+    }
+    
+    tables[internalId] = 1;
+  }
+  
+  void addFileSize(String file, long size) {
+    fileSizes.put(file, size);
+  }
+  
+  Map<List<String>,Long> calculateUsage() {
+
+    Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
+    
+    for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
+      log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
+      List<Integer> key = Arrays.asList(entry.getValue());
+      Long size = fileSizes.get(entry.getKey());
+      
+      Long tablesUsage = usage.get(key);
+      if (tablesUsage == null)
+        tablesUsage = 0l;
+      
+      tablesUsage += size;
+      
+      usage.put(key, tablesUsage);
+      
+    }
+    
+    Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
+    
+    for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
+      List<String> externalKey = new ArrayList<String>();
+      List<Integer> key = entry.getKey();
+      for (int i = 0; i < key.size(); i++)
+        if (key.get(i) != 0)
+          externalKey.add(externalIds.get(i));
+      
+      externalUsage.put(externalKey, entry.getValue());
+    }
+    
+    return externalUsage;
+  }
+  
+  public interface Printer {
+    void print(String line);
+  }
+  
+  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String>
tables, FileSystem fs, Connector conn, boolean humanReadable) 
+      throws TableNotFoundException,
+      IOException {
+    printDiskUsage(acuConf, tables, fs, conn, new Printer() {
+      @Override
+      public void print(String line) {
+        System.out.println(line);
+      }
+    }, humanReadable);
+  }
+  
+  public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration
acuConf, Set<String> tableIds, FileSystem fs, Connector conn)
+      throws IOException {
+    TableDiskUsage tdu = new TableDiskUsage();
+    
+    for (String tableId : tableIds)
+      tdu.addTable(tableId);
+    
+    HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
+    HashSet<String> emptyTableIds = new HashSet<String>();
+    
+    for (String tableId : tableIds) {
+      Scanner mdScanner = null;
+      try {
+        mdScanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+      } catch (TableNotFoundException e) {
+        throw new RuntimeException(e);
+      }
+      mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+      mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+      
+      if (!mdScanner.iterator().hasNext()) {
+        emptyTableIds.add(tableId);
+      }
+      
+      for (Entry<Key,Value> entry : mdScanner) {
+        String file = entry.getKey().getColumnQualifier().toString();
+        String parts[] = file.split("/");
+        String uniqueName = parts[parts.length - 1];
+        if (file.contains(":") || file.startsWith("../")) {
+          String ref = parts[parts.length - 3];
+          if (!ref.equals(tableId)) {
+            tablesReferenced.add(ref);
+          }
+        }
+        
+        tdu.linkFileAndTable(tableId, uniqueName);
+      }
+    }
+    
+    for (String tableId : tablesReferenced) {
+      for (String tableDir : ServerConstants.getTablesDirs()) {
+        FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+        if (files != null) {
+          for (FileStatus fileStatus : files) {
+            // Assumes that all filenames are unique
+            String name = fileStatus.getPath().getName();
+            tdu.addFileSize(name, fileStatus.getLen());
+          }
+        }
+      }
+    }
+    
+    HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
+    for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
+      reverseTableIdMap.put(entry.getValue(), entry.getKey());
+    
+    TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new
Comparator<TreeSet<String>>() {
+      
+      @Override
+      public int compare(TreeSet<String> o1, TreeSet<String> o2) {
+        int len1 = o1.size();
+        int len2 = o2.size();
+        
+        int min = Math.min(len1, len2);
+        
+        Iterator<String> iter1 = o1.iterator();
+        Iterator<String> iter2 = o2.iterator();
+        
+        int count = 0;
+        
+        while (count < min) {
+          String s1 = iter1.next();
+          String s2 = iter2.next();
+          
+          int cmp = s1.compareTo(s2);
+          
+          if (cmp != 0)
+            return cmp;
+          
+          count++;
+        }
+        
+        return len1 - len2;
+      }
+    });
+    
+    for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+      TreeSet<String> tableNames = new TreeSet<String>();
+      for (String tableId : entry.getKey())
+        tableNames.add(reverseTableIdMap.get(tableId));
+      
+      usage.put(tableNames, entry.getValue());
+    }
+    
+    if (!emptyTableIds.isEmpty()) {
+      TreeSet<String> emptyTables = new TreeSet<String>();
+      for (String tableId : emptyTableIds) {
+        emptyTables.add(reverseTableIdMap.get(tableId));
+      }
+      usage.put(emptyTables, 0L);
+    }
+    
+    return usage;
+  }
+  
+  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String>
tables, FileSystem fs, Connector conn, Printer printer, boolean humanReadable)
+      throws TableNotFoundException, IOException {
+    
+    HashSet<String> tableIds = new HashSet<String>();
+    
+    for (String tableName : tables) {
+      String tableId = conn.tableOperations().tableIdMap().get(tableName);
+      if (tableId == null)
+        throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
+      
+      tableIds.add(tableId);
+    }
+    
+    Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+
+    String valueFormat = humanReadable ? "%9s" : "%,24d";
+    for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
+      Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
+      printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
+    }
+  }
+
+  
   static class Opts extends ClientOpts {
     @Parameter(description=" <table> { <table> ... } ")
     List<String> tables = new ArrayList<String>();
   }
-  
+    
   /**
    * @param args
    */
@@ -42,7 +280,7 @@ public class TableDiskUsage {
     Opts opts = new Opts();
     opts.parseArgs(TableDiskUsage.class.getName(), args);
     Connector conn = opts.getConnector();
-    org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(),
opts.tables, fs, conn, false);
+    org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(),
opts.tables, fs, conn, false);
   }
   
 }

Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java
Mon Jun 17 13:26:43 2013
@@ -95,7 +95,7 @@ public class MultiReaderTest {
   
   @Test
   public void testMultiReader() throws IOException {
-    String manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps").toString();
+    Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
     MultiReader reader = new MultiReader(fs, manyMaps);
     IntWritable key = new IntWritable();
     BytesWritable value = new BytesWritable();

Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
Mon Jun 17 13:26:43 2013
@@ -118,7 +118,7 @@ public class SortedLogRecoveryTest {
     final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir";
     FileSystem fs = FileSystemImpl.getLocal();
     fs.deleteRecursively(new Path(workdir));
-    ArrayList<String> dirs = new ArrayList<String>();
+    ArrayList<Path> dirs = new ArrayList<Path>();
     try {
       for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
         String path = workdir + "/" + entry.getKey();
@@ -129,7 +129,7 @@ public class SortedLogRecoveryTest {
         }
         map.close();
         ns.create(new Path(path, "finished")).close();
-        dirs.add(path);
+        dirs.add(new Path(path));
       }
       // Recover
       SortedLogRecovery recovery = new SortedLogRecovery(fs);

Modified: accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
(original)
+++ accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
Mon Jun 17 13:26:43 2013
@@ -43,8 +43,11 @@ import org.apache.accumulo.fate.zookeepe
 import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
 import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.FileSystemImpl;
 import org.apache.accumulo.server.master.state.Assignment;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.accumulo.server.security.SecurityConstants;
@@ -134,7 +137,7 @@ public class SplitRecoveryTest extends F
     for (int i = 0; i < extents.length; i++) {
       KeyExtent extent = extents[i];
       
-      String tdir = "/dir_" + i;
+      String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString()
+ "/dir_" + i;
       MetadataTable.addTablet(extent, tdir, SecurityConstants.getSystemCredentials(), TabletTime.LOGICAL_TIME_ID,
zl);
       SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
       mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017
+ i, 10000 + i));



Mime
View raw message