Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 02C6FF408 for ; Fri, 3 May 2013 00:03:16 +0000 (UTC) Received: (qmail 52070 invoked by uid 500); 3 May 2013 00:03:15 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 52039 invoked by uid 500); 3 May 2013 00:03:15 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 52032 invoked by uid 99); 3 May 2013 00:03:15 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 May 2013 00:03:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 May 2013 00:03:13 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A46482388A29; Fri, 3 May 2013 00:02:53 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1478613 [4/4] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ core/src/main/ja... Date: Fri, 03 May 2013 00:02:52 -0000 To: commits@accumulo.apache.org From: kturner@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130503000253.A46482388A29@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/TableDiskUsage.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/TableDiskUsage.java?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/TableDiskUsage.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/TableDiskUsage.java Fri May 3 00:02:52 2013 @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -127,20 +128,10 @@ public class TableDiskUsage { }, humanReadable); } - public static Map,Long> getDiskUsage(AccumuloConfiguration acuConf, Collection tables, FileSystem fs, Connector conn, boolean humanReadable) - throws TableNotFoundException, IOException { + public static Map,Long> getDiskUsage(AccumuloConfiguration acuConf, Set tableIds, FileSystem fs, Connector conn, boolean humanReadable) + throws IOException { TableDiskUsage tdu = new TableDiskUsage(); - HashSet tableIds = new HashSet(); - - for (String tableName : tables) { - String tableId = conn.tableOperations().tableIdMap().get(tableName); - if (tableId == null) - throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found"); - - tableIds.add(tableId); - } - for (String tableId : tableIds) tdu.addTable(tableId); @@ -148,7 +139,12 @@ public class TableDiskUsage { HashSet emptyTableIds = new HashSet(); for (String tableId : tableIds) { - Scanner mdScanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + Scanner mdScanner = null; + try { + mdScanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY); mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); @@ -236,7 +232,17 @@ public class TableDiskUsage { public static void printDiskUsage(AccumuloConfiguration acuConf, Collection tables, FileSystem fs, Connector conn, Printer printer, boolean humanReadable) throws TableNotFoundException, IOException { - Map,Long> usage = getDiskUsage(acuConf, tables, fs, conn, humanReadable); + HashSet tableIds = new HashSet(); + + for (String tableName : tables) { + String tableId = conn.tableOperations().tableIdMap().get(tableName); + if (tableId == null) + throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found"); + + tableIds.add(tableId); + } + + Map,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn, humanReadable); String valueFormat = humanReadable ? "%9s" : "%,24d"; for (Entry,Long> entry : usage.entrySet()) { Modified: accumulo/trunk/core/src/main/thrift/client.thrift URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/thrift/client.thrift?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/core/src/main/thrift/client.thrift (original) +++ accumulo/trunk/core/src/main/thrift/client.thrift Fri May 3 00:02:52 2013 @@ -90,6 +90,11 @@ exception ThriftTableOperationException 5:string description } +struct TDiskUsage { + 1:list tables + 2:i64 usage +} + service ClientService { // system management methods @@ -103,6 +108,8 @@ service ClientService { void ping(2:security.TCredentials credentials) throws (1:ThriftSecurityException sec) + list getDiskUsage(2:set tables, 1:security.TCredentials credentials) throws (1:ThriftSecurityException sec, 2:ThriftTableOperationException toe) + // user management methods set listLocalUsers(2:trace.TInfo tinfo, 3:security.TCredentials credentials) throws (1:ThriftSecurityException sec) void createLocalUser(5:trace.TInfo tinfo, 6:security.TCredentials credentials, 2:string principal, 3:binary password) throws (1:ThriftSecurityException sec) Modified: accumulo/trunk/core/src/main/thrift/tabletserver.thrift URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/thrift/tabletserver.thrift?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/core/src/main/thrift/tabletserver.thrift (original) +++ accumulo/trunk/core/src/main/thrift/tabletserver.thrift Fri May 3 00:02:52 2013 @@ -87,11 +87,6 @@ struct ActiveScan { 13:list authorizations } -struct DiskUsage { - 1:list tables - 2:i64 usage -} - enum CompactionType { MINOR, MERGE, @@ -192,7 +187,6 @@ service TabletClientService extends clie list getActiveScans(2:trace.TInfo tinfo, 1:security.TCredentials credentials) throws (1:client.ThriftSecurityException sec) list getActiveCompactions(2:trace.TInfo tinfo, 1:security.TCredentials credentials) throws (1:client.ThriftSecurityException sec) - list getDiskUsage(2:set tables, 1:security.TCredentials credentials) throws (1:client.ThriftSecurityException sec, 2:client.ThriftTableOperationException toe) oneway void removeLogs(1:trace.TInfo tinfo, 2:security.TCredentials credentials, 3:list filenames) } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java Fri May 3 00:02:52 2013 @@ -16,21 +16,28 @@ */ package org.apache.accumulo.server.client; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.Callable; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.impl.thrift.ClientService; import org.apache.accumulo.core.client.impl.thrift.ConfigurationType; import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode; +import org.apache.accumulo.core.client.impl.thrift.TDiskUsage; import org.apache.accumulo.core.client.impl.thrift.TableOperation; import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType; import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; @@ -38,17 +45,21 @@ import org.apache.accumulo.core.client.i import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.TableDiskUsage; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.security.AuditedSecurityOperation; import org.apache.accumulo.server.security.SecurityOperation; import org.apache.accumulo.server.zookeeper.TransactionWatcher; import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.accumulo.trace.thrift.TInfo; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; import org.apache.thrift.TException; @@ -308,4 +319,39 @@ public class ClientServiceHandler implem return false; } } + + @Override + public List getDiskUsage(Set tables, TCredentials credentials) throws ThriftTableOperationException, ThriftSecurityException, TException { + try { + Connector conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)); + + HashSet tableIds = new HashSet(); + + for (String table : tables) { + // ensure that table table exists + String tableId = checkTableId(table, null); + tableIds.add(tableId); + if (!security.canScan(credentials, tableId)) + throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); + } + + AccumuloConfiguration conf = new ServerConfiguration(instance).getConfiguration(); + FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), conf); + + // use the same set of tableIds that were validated above to avoid race conditions + Map,Long> diskUsage = TableDiskUsage.getDiskUsage(new ServerConfiguration(instance).getConfiguration(), tableIds, fs, conn, false); + List retUsages = new ArrayList(); + for (Map.Entry,Long> usageItem : diskUsage.entrySet()) { + retUsages.add(new TDiskUsage(new ArrayList(usageItem.getKey()), usageItem.getValue())); + } + return retUsages; + + } catch (AccumuloSecurityException e) { + throw e.asThriftException(); + } catch (AccumuloException e) { + throw new TException(e); + } catch (IOException e) { + throw new TException(e); + } + } } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Fri May 3 00:02:52 2013 @@ -67,16 +67,12 @@ import javax.management.StandardMBean; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.impl.ScannerImpl; import org.apache.accumulo.core.client.impl.TabletType; import org.apache.accumulo.core.client.impl.Translator; import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode; -import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType; import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; -import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.constraints.Constraint.Environment; @@ -109,14 +105,11 @@ import org.apache.accumulo.core.master.t import org.apache.accumulo.core.master.thrift.TabletLoadState; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.SecurityUtil; -import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction; import org.apache.accumulo.core.tabletserver.thrift.ActiveScan; import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException; -import org.apache.accumulo.core.tabletserver.thrift.DiskUsage; import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException; import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException; import org.apache.accumulo.core.tabletserver.thrift.ScanState; @@ -136,7 +129,6 @@ import org.apache.accumulo.core.util.Ser import org.apache.accumulo.core.util.ServerServices.Service; import org.apache.accumulo.core.util.SimpleThreadPool; import org.apache.accumulo.core.util.Stat; -import org.apache.accumulo.core.util.TableDiskUsage; import org.apache.accumulo.core.util.ThriftUtil; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.zookeeper.ZooUtil; @@ -2117,34 +2109,7 @@ public class TabletServer extends Abstra } } } - - @Override - public List getDiskUsage(Set tables, TCredentials credentials) throws ThriftTableOperationException, ThriftSecurityException, TException { - try { - Connector conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)); - for (String table : tables) { - if (conn.tableOperations().exists(table) && !conn.securityOperations().hasTablePermission(credentials.getPrincipal(), table, TablePermission.READ)) - throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); - } - - Map,Long> diskUsage = TableDiskUsage.getDiskUsage(getServerConfig().getConfiguration(), tables, fs, conn, false); - List retUsages = new ArrayList(); - for (Map.Entry,Long> usageItem : diskUsage.entrySet()) { - retUsages.add(new DiskUsage(new ArrayList(usageItem.getKey()), usageItem.getValue())); - } - return retUsages; - - } catch (AccumuloSecurityException e) { - // *.security.thrift.SecurityErrorCode is deprecated- let's assume the two stay in sync... - throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.valueOf(e.getSecurityErrorCode().name())); - } catch (TableNotFoundException e) { - throw new ThriftTableOperationException(null, e.getTableName(), null, TableOperationExceptionType.NOTFOUND, "Table was not found"); - } catch (Exception e) { - throw new TException(e); - } - } - @Override public List getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException { try { Modified: accumulo/trunk/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java (original) +++ accumulo/trunk/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java Fri May 3 00:02:52 2013 @@ -23,11 +23,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException; -import org.apache.accumulo.core.tabletserver.thrift.DiskUsage; -import org.apache.accumulo.trace.thrift.TInfo; import org.apache.accumulo.core.cli.Help; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.ZooKeeperInstance; @@ -69,6 +65,7 @@ import org.apache.accumulo.server.master import org.apache.accumulo.server.security.SecurityConstants; import org.apache.accumulo.server.util.TServerUtils; import org.apache.accumulo.server.zookeeper.TransactionWatcher; +import org.apache.accumulo.trace.thrift.TInfo; import org.apache.hadoop.io.Text; import org.apache.thrift.TException; @@ -211,11 +208,6 @@ public class NullTserver { public List getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException { return new ArrayList(); } - - @Override - public List getDiskUsage(Set tables, TCredentials credentials) throws ThriftSecurityException, ThriftTableOperationException, TException { - return null; - } } static class Opts extends Help { Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java?rev=1478613&r1=1478612&r2=1478613&view=diff ============================================================================== --- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java (original) +++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java Fri May 3 00:02:52 2013 @@ -17,16 +17,21 @@ package org.apache.accumulo.test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; @@ -35,8 +40,11 @@ import org.apache.accumulo.core.client.a import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.constraints.DefaultKeySizeConstraint; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; +import org.apache.hadoop.io.Text; import org.apache.thrift.TException; import org.apache.thrift.transport.TTransportException; import org.junit.AfterClass; @@ -67,7 +75,7 @@ public class TableOperationsIT { } @Test - public void getDiskUsage() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TException { + public void getDiskUsageErrors() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TException { connector.tableOperations().create("table1"); List diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton("table1")); assertEquals(1, diskUsage.size()); @@ -88,6 +96,61 @@ public class TableOperationsIT { } @Test + public void getDiskUsage() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TException { + + connector.tableOperations().create("table1"); + + // verify 0 disk usage + List diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton("table1")); + assertEquals(1, diskUsages.size()); + assertEquals(1, diskUsages.get(0).getTables().size()); + assertEquals(new Long(0), diskUsages.get(0).getUsage()); + assertEquals("table1", diskUsages.get(0).getTables().first()); + + // add some data + BatchWriter bw = connector.createBatchWriter("table1", new BatchWriterConfig()); + Mutation m = new Mutation("a"); + m.put("b", "c", new Value("abcde".getBytes())); + bw.addMutation(m); + bw.flush(); + bw.close(); + + connector.tableOperations().compact("table1", new Text("A"), new Text("z"), true, true); + + // verify we have usage + diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton("table1")); + assertEquals(1, diskUsages.size()); + assertEquals(1, diskUsages.get(0).getTables().size()); + assertTrue(diskUsages.get(0).getUsage() > 0); + assertEquals("table1", diskUsages.get(0).getTables().first()); + + // clone table + connector.tableOperations().clone("table1", "table2", false, null, null); + + // verify tables are exactly the same + Set tables = new HashSet(); + tables.add("table1"); + tables.add("table2"); + diskUsages = connector.tableOperations().getDiskUsage(tables); + assertEquals(1, diskUsages.size()); + assertEquals(2, diskUsages.get(0).getTables().size()); + assertTrue(diskUsages.get(0).getUsage() > 0); + + connector.tableOperations().compact("table1", new Text("A"), new Text("z"), true, true); + connector.tableOperations().compact("table2", new Text("A"), new Text("z"), true, true); + + // verify tables have differences + diskUsages = connector.tableOperations().getDiskUsage(tables); + assertEquals(2, diskUsages.size()); + assertEquals(1, diskUsages.get(0).getTables().size()); + assertEquals(1, diskUsages.get(1).getTables().size()); + assertTrue(diskUsages.get(0).getUsage() > 0); + assertTrue(diskUsages.get(1).getUsage() > 0); + + connector.tableOperations().delete("table1"); + } + + @Test public void createTable() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException { connector.tableOperations().create("table1"); Iterable> itrProps = connector.tableOperations().getProperties("table1");