Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 46AE310B73 for ; Tue, 25 Nov 2014 22:36:46 +0000 (UTC) Received: (qmail 50466 invoked by uid 500); 25 Nov 2014 22:36:46 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 50417 invoked by uid 500); 25 Nov 2014 22:36:46 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 50309 invoked by uid 99); 25 Nov 2014 22:36:46 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Nov 2014 22:36:46 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A6BBAA1AAAE; Tue, 25 Nov 2014 22:36:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Tue, 25 Nov 2014 22:36:50 -0000 Message-Id: In-Reply-To: <8e88d34caa1e4ba0a2674c2ab43d0f36@git.apache.org> References: <8e88d34caa1e4ba0a2674c2ab43d0f36@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/10] accumulo git commit: ACCUMULO-3199 Internal refactor to add ClientContext http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index a15e05e..670c541 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@ -80,15 +80,16 @@ import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.server.Accumulo; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.constraints.MetadataConstraints; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter; import org.apache.accumulo.server.replication.StatusCombiner; import org.apache.accumulo.server.security.AuditedSecurityOperation; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.tables.TableManager; import org.apache.accumulo.server.tablets.TabletTime; import org.apache.accumulo.server.util.ReplicationTableUtil; @@ -546,7 +547,13 @@ public class Initialize { } private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException, ThriftSecurityException { - AuditedSecurityOperation.getInstance(iid, true).initializeSecurity(SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()), DEFAULT_ROOT_USER, + AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()) { + @Override + public synchronized AccumuloConfiguration getConfiguration() { + return getSiteConfiguration(); + } + }); + AuditedSecurityOperation.getInstance(context, true).initializeSecurity(context.rpcCreds(), DEFAULT_ROOT_USER, opts.rootpass); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java index 3388b5b..d392bde 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java @@ -27,9 +27,8 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException; @@ -40,7 +39,6 @@ import org.apache.accumulo.core.util.ServerServices; import org.apache.accumulo.core.util.ThriftUtil; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.server.master.state.TServerInstance; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.util.Halt; import org.apache.accumulo.server.util.time.SimpleTimer; import org.apache.accumulo.server.zookeeper.ZooCache; @@ -68,8 +66,7 @@ public class LiveTServerSet implements Watcher { private static final Logger log = Logger.getLogger(LiveTServerSet.class); private final Listener cback; - private final Instance instance; - private final AccumuloConfiguration conf; + private final ClientContext context; private ZooCache zooCache; public class TServerConnection { @@ -80,22 +77,22 @@ public class LiveTServerSet implements Watcher { } private String lockString(ZooLock mlock) { - return mlock.getLockID().serialize(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK); + return mlock.getLockID().serialize(ZooUtil.getRoot(context.getInstance()) + Constants.ZMASTER_LOCK); } public void assignTablet(ZooLock lock, KeyExtent extent) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.loadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift()); + client.loadTablet(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), extent.toThrift()); } finally { ThriftUtil.returnClient(client); } } public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.unloadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift(), save); + client.unloadTablet(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), extent.toThrift(), save); } finally { ThriftUtil.returnClient(client); } @@ -106,11 +103,11 @@ public class LiveTServerSet implements Watcher { if (usePooledConnection == true) throw new UnsupportedOperationException(); - TTransport transport = ThriftUtil.createTransport(address, conf); + TTransport transport = ThriftUtil.createTransport(address, context); try { TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport); - return client.getTabletServerStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance)); + return client.getTabletServerStatus(Tracer.traceInfo(), context.rpcCreds()); } finally { if (transport != null) transport.close(); @@ -118,73 +115,72 @@ public class LiveTServerSet implements Watcher { } public void halt(ZooLock lock) throws TException, ThriftSecurityException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.halt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock)); + client.halt(Tracer.traceInfo(), context.rpcCreds(), lockString(lock)); } finally { ThriftUtil.returnClient(client); } } public void fastHalt(ZooLock lock) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.fastHalt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock)); + client.fastHalt(Tracer.traceInfo(), context.rpcCreds(), lockString(lock)); } finally { ThriftUtil.returnClient(client); } } public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.flush(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId, - startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow)); + client.flush(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), tableId, startRow == null ? null : ByteBuffer.wrap(startRow), + endRow == null ? null : ByteBuffer.wrap(endRow)); } finally { ThriftUtil.returnClient(client); } } public void chop(ZooLock lock, KeyExtent extent) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.chop(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift()); + client.chop(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), extent.toThrift()); } finally { ThriftUtil.returnClient(client); } } public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.splitTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), extent.toThrift(), - ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength())); + client.splitTablet(Tracer.traceInfo(), context.rpcCreds(), extent.toThrift(), ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength())); } finally { ThriftUtil.returnClient(client); } } public void flushTablet(ZooLock lock, KeyExtent extent) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.flushTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift()); + client.flushTablet(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), extent.toThrift()); } finally { ThriftUtil.returnClient(client); } } public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { - client.compact(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId, - startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow)); + client.compact(Tracer.traceInfo(), context.rpcCreds(), lockString(lock), tableId, startRow == null ? null : ByteBuffer.wrap(startRow), + endRow == null ? null : ByteBuffer.wrap(endRow)); } finally { ThriftUtil.returnClient(client); } } public boolean isActive(long tid) throws TException { - TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf); + TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context); try { return client.isActive(Tracer.traceInfo(), tid); } finally { @@ -212,11 +208,9 @@ public class LiveTServerSet implements Watcher { // The set of entries in zookeeper without locks, and the first time each was noticed private Map locklessServers = new HashMap(); - public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) { + public LiveTServerSet(ClientContext context, Listener cback) { this.cback = cback; - this.instance = instance; - this.conf = conf; - + this.context = context; } public synchronized ZooCache getZooCache() { @@ -227,7 +221,7 @@ public class LiveTServerSet implements Watcher { public synchronized void startListeningForTabletServerChanges() { scanServers(); - SimpleTimer.getInstance(conf).schedule(new Runnable() { + SimpleTimer.getInstance(context.getConfiguration()).schedule(new Runnable() { @Override public void run() { scanServers(); @@ -240,7 +234,7 @@ public class LiveTServerSet implements Watcher { final Set updates = new HashSet(); final Set doomed = new HashSet(); - final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS; + final String path = ZooUtil.getRoot(context.getInstance()) + Constants.ZTSERVERS; HashSet all = new HashSet(current.keySet()); all.addAll(getZooCache().getChildren(path)); @@ -333,7 +327,7 @@ public class LiveTServerSet implements Watcher { final Set updates = new HashSet(); final Set doomed = new HashSet(); - final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS; + final String path = ZooUtil.getRoot(context.getInstance()) + Constants.ZTSERVERS; try { checkServer(updates, doomed, path, server); @@ -387,7 +381,7 @@ public class LiveTServerSet implements Watcher { currentInstances.remove(server); log.info("Removing zookeeper lock for " + server); - String fullpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + zPath; + String fullpath = ZooUtil.getRoot(context.getInstance()) + Constants.ZTSERVERS + "/" + zPath; try { ZooReaderWriter.getInstance().recursiveDelete(fullpath, SKIP); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java index d96f9b0..5eae890 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java @@ -34,7 +34,6 @@ import org.apache.accumulo.core.master.state.tables.TableState; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletMigration; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.tables.TableManager; import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.log4j.Logger; @@ -133,7 +132,7 @@ public class TableLoadBalancer extends TabletBalancer { protected TableOperations getTableOperations() { if (tops == null) try { - tops = configuration.getInstance().getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()).tableOperations(); + tops = context.getConnector().tableOperations(); } catch (AccumuloException e) { log.error("Unable to access table operations from within table balancer", e); } catch (AccumuloSecurityException e) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java index 30a2460..cd25d49 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java @@ -31,11 +31,11 @@ import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client; import org.apache.accumulo.core.tabletserver.thrift.TabletStats; import org.apache.accumulo.core.trace.Tracer; import org.apache.accumulo.core.util.ThriftUtil; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletMigration; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.log4j.Logger; import org.apache.thrift.TException; import org.apache.thrift.transport.TTransportException; @@ -48,15 +48,18 @@ public abstract class TabletBalancer { protected ServerConfigurationFactory configuration; + protected AccumuloServerContext context; + /** * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration. */ public void init(ServerConfigurationFactory conf) { + context = new AccumuloServerContext(conf); configuration = conf; } public void init(ServerConfiguration conf) { - configuration = (ServerConfigurationFactory)conf; + init((ServerConfigurationFactory) conf); } /** @@ -191,11 +194,9 @@ public abstract class TabletBalancer { */ public List getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException { log.debug("Scanning tablet server " + tserver + " for table " + tableId); - Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration()); + Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), context); try { - List onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(configuration.getInstance()), - tableId); - return onlineTabletsForTable; + return client.getTabletStats(Tracer.traceInfo(), context.rpcCreds(), tableId); } catch (TTransportException e) { log.error("Unable to connect to " + tserver + ": " + e); } finally { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java index 63617d6..d57a3ef 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java @@ -21,15 +21,13 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; -import org.apache.accumulo.core.security.Credentials; -import org.apache.accumulo.server.client.HdfsZooInstance; -import org.apache.accumulo.server.security.SystemCredentials; +import org.apache.accumulo.server.AccumuloServerContext; public class MetaDataStateStore extends TabletStateStore { // private static final Logger log = Logger.getLogger(MetaDataStateStore.class); @@ -38,33 +36,31 @@ public class MetaDataStateStore extends TabletStateStore { private static final int LATENCY = 1000; private static final int MAX_MEMORY = 200 * 1024 * 1024; - final protected Instance instance; + final protected ClientContext context; final protected CurrentState state; - final protected Credentials credentials; final private String targetTableName; - protected MetaDataStateStore(Instance instance, Credentials credentials, CurrentState state, String targetTableName) { - this.instance = instance; + protected MetaDataStateStore(ClientContext context, CurrentState state, String targetTableName) { + this.context = context; this.state = state; - this.credentials = credentials; this.targetTableName = targetTableName; } - public MetaDataStateStore(Instance instance, Credentials credentials, CurrentState state) { - this(instance, credentials, state, MetadataTable.NAME); + public MetaDataStateStore(ClientContext context, CurrentState state) { + this(context, state, MetadataTable.NAME); } - protected MetaDataStateStore(String tableName) { - this(HdfsZooInstance.getInstance(), SystemCredentials.get(), null, tableName); + protected MetaDataStateStore(AccumuloServerContext context, String tableName) { + this(context, null, tableName); } - public MetaDataStateStore() { - this(MetadataTable.NAME); + public MetaDataStateStore(AccumuloServerContext context) { + this(context, MetadataTable.NAME); } @Override public ClosableIterator iterator() { - return new MetaDataTableScanner(instance, credentials, MetadataSchema.TabletsSection.getRange(), state); + return new MetaDataTableScanner(context, MetadataSchema.TabletsSection.getRange(), state); } @Override @@ -90,7 +86,7 @@ public class MetaDataStateStore extends TabletStateStore { BatchWriter createBatchWriter() { try { - return instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createBatchWriter(targetTableName, + return context.getConnector().createBatchWriter(targetTableName, new BatchWriterConfig().setMaxMemory(MAX_MEMORY).setMaxLatency(LATENCY, TimeUnit.MILLISECONDS).setMaxWriteThreads(THREADS)); } catch (TableNotFoundException e) { // ya, I don't think so http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java index 155a5d2..d318ccc 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java @@ -28,9 +28,9 @@ import java.util.SortedMap; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.ScannerBase; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Range; @@ -41,7 +41,6 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.Credentials; import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; @@ -52,14 +51,14 @@ public class MetaDataTableScanner implements ClosableIterator> iter = null; - public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state) { - this(instance, credentials, range, state, MetadataTable.NAME); + public MetaDataTableScanner(ClientContext context, Range range, CurrentState state) { + this(context, range, state, MetadataTable.NAME); } - MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state, String tableName) { + MetaDataTableScanner(ClientContext context, Range range, CurrentState state, String tableName) { // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables try { - Connector connector = instance.getConnector(credentials.getPrincipal(), credentials.getToken()); + Connector connector = context.getConnector(); mdScanner = connector.createBatchScanner(tableName, Authorizations.EMPTY, 8); configureScanner(mdScanner, state); mdScanner.setRanges(Collections.singletonList(range)); @@ -90,14 +89,15 @@ public class MetaDataTableScanner implements ClosableIterator iterator() { - return new MetaDataTableScanner(instance, credentials, MetadataSchema.TabletsSection.getRange(), state, RootTable.NAME); + return new MetaDataTableScanner(context, MetadataSchema.TabletsSection.getRange(), state, RootTable.NAME); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java index 7c75454..41de64a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java @@ -19,6 +19,8 @@ package org.apache.accumulo.server.master.state; import java.util.Collection; import java.util.Collections; +import org.apache.accumulo.server.AccumuloServerContext; + /** * Interface for storing information about tablet assignments. There are three implementations: * @@ -57,26 +59,26 @@ public abstract class TabletStateStore implements Iterable */ abstract public void unassign(Collection tablets) throws DistributedStoreException; - public static void unassign(TabletLocationState tls) throws DistributedStoreException { + public static void unassign(AccumuloServerContext context, TabletLocationState tls) throws DistributedStoreException { TabletStateStore store; if (tls.extent.isRootTablet()) { store = new ZooTabletStateStore(); } else if (tls.extent.isMeta()) { - store = new RootTabletStateStore(); + store = new RootTabletStateStore(context); } else { - store = new MetaDataStateStore(); + store = new MetaDataStateStore(context); } store.unassign(Collections.singletonList(tls)); } - public static void setLocation(Assignment assignment) throws DistributedStoreException { + public static void setLocation(AccumuloServerContext context, Assignment assignment) throws DistributedStoreException { TabletStateStore store; if (assignment.tablet.isRootTablet()) { store = new ZooTabletStateStore(); } else if (assignment.tablet.isMeta()) { - store = new RootTabletStateStore(); + store = new RootTabletStateStore(context); } else { - store = new MetaDataStateStore(); + store = new MetaDataStateStore(context); } store.setLocations(Collections.singletonList(assignment)); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReport.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReport.java b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReport.java index 99349b1..0dfe9c0 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReport.java +++ b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReport.java @@ -17,6 +17,7 @@ package org.apache.accumulo.server.problems; import static com.google.common.base.Preconditions.checkNotNull; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -35,8 +36,8 @@ import org.apache.accumulo.core.util.Encoding; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.client.HdfsZooInstance; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.hadoop.io.Text; @@ -135,21 +136,22 @@ public class ProblemReport { } } - void removeFromMetadataTable() throws Exception { + void removeFromMetadataTable(AccumuloServerContext context) throws Exception { Mutation m = new Mutation(new Text("~err_" + tableName)); m.putDelete(new Text(problemType.name()), new Text(resource)); - MetadataTableUtil.getMetadataTable(SystemCredentials.get()).update(m); + MetadataTableUtil.getMetadataTable(context).update(m); } - void saveToMetadataTable() throws Exception { + void saveToMetadataTable(AccumuloServerContext context) throws Exception { Mutation m = new Mutation(new Text("~err_" + tableName)); m.put(new Text(problemType.name()), new Text(resource), new Value(encode())); - MetadataTableUtil.getMetadataTable(SystemCredentials.get()).update(m); + MetadataTableUtil.getMetadataTable(context).update(m); } void removeFromZooKeeper() throws Exception { removeFromZooKeeper(ZooReaderWriter.getInstance(), HdfsZooInstance.getInstance()); } + void removeFromZooKeeper(ZooReaderWriter zoorw, Instance instance) throws IOException, KeeperException, InterruptedException { String zpath = getZPath(instance); zoorw.recursiveDelete(zpath, NodeMissingPolicy.SKIP); @@ -158,6 +160,7 @@ public class ProblemReport { void saveToZooKeeper() throws Exception { saveToZooKeeper(ZooReaderWriter.getInstance(), HdfsZooInstance.getInstance()); } + void saveToZooKeeper(ZooReaderWriter zoorw, Instance instance) throws IOException, KeeperException, InterruptedException { zoorw.putPersistentData(getZPath(instance), encode(), NodeExistsPolicy.OVERWRITE); } @@ -178,6 +181,7 @@ public class ProblemReport { static ProblemReport decodeZooKeeperEntry(String node) throws Exception { return decodeZooKeeperEntry(node, ZooReaderWriter.getInstance(), HdfsZooInstance.getInstance()); } + static ProblemReport decodeZooKeeperEntry(String node, ZooReaderWriter zoorw, Instance instance) throws IOException, KeeperException, InterruptedException { byte bytes[] = Encoding.decodeBase64FileName(node); http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java index 69d73e1..51ac12f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java @@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.system.InterruptibleIterator; +import org.apache.accumulo.server.AccumuloServerContext; public class ProblemReportingIterator implements InterruptibleIterator { private SortedKeyValueIterator source; @@ -35,8 +36,11 @@ public class ProblemReportingIterator implements InterruptibleIterator { private boolean continueOnError; private String resource; private String table; + private final AccumuloServerContext context; - public ProblemReportingIterator(String table, String resource, boolean continueOnError, SortedKeyValueIterator source) { + public ProblemReportingIterator(AccumuloServerContext context, String table, String resource, boolean continueOnError, + SortedKeyValueIterator source) { + this.context = context; this.table = table; this.resource = resource; this.continueOnError = continueOnError; @@ -45,7 +49,7 @@ public class ProblemReportingIterator implements InterruptibleIterator { @Override public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { - return new ProblemReportingIterator(table, resource, continueOnError, source.deepCopy(env)); + return new ProblemReportingIterator(context, table, resource, continueOnError, source.deepCopy(env)); } @Override @@ -77,7 +81,7 @@ public class ProblemReportingIterator implements InterruptibleIterator { source.next(); } catch (IOException ioe) { sawError = true; - ProblemReports.getInstance().report(new ProblemReport(table, ProblemType.FILE_READ, resource, ioe)); + ProblemReports.getInstance(context).report(new ProblemReport(table, ProblemType.FILE_READ, resource, ioe)); if (!continueOnError) { throw ioe; } @@ -94,7 +98,7 @@ public class ProblemReportingIterator implements InterruptibleIterator { source.seek(range, columnFamilies, inclusive); } catch (IOException ioe) { sawError = true; - ProblemReports.getInstance().report(new ProblemReport(table, ProblemType.FILE_READ, resource, ioe)); + ProblemReports.getInstance(context).report(new ProblemReport(table, ProblemType.FILE_READ, resource, ioe)); if (!continueOnError) { throw ioe; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReports.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReports.java b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReports.java index 23d4de5..a559985 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReports.java +++ b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReports.java @@ -46,8 +46,9 @@ import org.apache.accumulo.core.util.LoggingRunnable; import org.apache.accumulo.core.util.NamingThreadFactory; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.client.HdfsZooInstance; -import org.apache.accumulo.server.security.SystemCredentials; +import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.commons.collections.map.LRUMap; @@ -69,6 +70,12 @@ public class ProblemReports implements Iterable { private ExecutorService reportExecutor = new ThreadPoolExecutor(0, 1, 60, TimeUnit.SECONDS, new LinkedBlockingQueue(500), new NamingThreadFactory( "acu-problem-reporter")); + private final AccumuloServerContext context; + + public ProblemReports(AccumuloServerContext context) { + this.context = context; + } + public void report(final ProblemReport pr) { synchronized (problemReports) { @@ -92,7 +99,7 @@ public class ProblemReports implements Iterable { pr.saveToZooKeeper(); } else { // file report in metadata table - pr.saveToMetadataTable(); + pr.saveToMetadataTable(context); } } catch (Exception e) { log.error("Failed to file problem report " + pr.getTableName() + " " + pr.getProblemType() + " " + pr.getResource(), e); @@ -128,7 +135,7 @@ public class ProblemReports implements Iterable { pr.removeFromZooKeeper(); } else { // file report in metadata table - pr.removeFromMetadataTable(); + pr.removeFromMetadataTable(context); } } catch (Exception e) { log.error("Failed to delete problem report " + pr.getTableName() + " " + pr.getProblemType() + " " + pr.getResource(), e); @@ -155,7 +162,7 @@ public class ProblemReports implements Iterable { return; } - Connector connector = HdfsZooInstance.getInstance().getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()); + Connector connector = context.getConnector(); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.addScanIterator(new IteratorSetting(1, "keys-only", SortedKeyIterator.class)); @@ -170,7 +177,7 @@ public class ProblemReports implements Iterable { } if (hasProblems) - MetadataTableUtil.getMetadataTable(SystemCredentials.get()).update(delMut); + MetadataTableUtil.getMetadataTable(context).update(delMut); } private static boolean isMeta(String tableId) { @@ -191,7 +198,7 @@ public class ProblemReports implements Iterable { try { List children; if (table == null || isMeta(table)) { - children = zoo.getChildren(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZPROBLEMS); + children = zoo.getChildren(ZooUtil.getRoot(context.getInstance()) + Constants.ZPROBLEMS); } else { children = Collections.emptyList(); } @@ -210,7 +217,7 @@ public class ProblemReports implements Iterable { if (iter2 == null) { try { if ((table == null || !isMeta(table)) && iter1Count == 0) { - Connector connector = HdfsZooInstance.getInstance().getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()); + Connector connector = context.getConnector(); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.setTimeout(3, TimeUnit.SECONDS); @@ -283,16 +290,16 @@ public class ProblemReports implements Iterable { return iterator(null); } - public static synchronized ProblemReports getInstance() { + public static synchronized ProblemReports getInstance(AccumuloServerContext context) { if (instance == null) { - instance = new ProblemReports(); + instance = new ProblemReports(context); } return instance; } public static void main(String args[]) throws Exception { - getInstance().printProblems(); + getInstance(new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()))).printProblems(); } public Map> summarize() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java index 50b15f5..edd2642 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java @@ -32,7 +32,7 @@ import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.replication.ReplicaSystem; import org.apache.accumulo.core.client.replication.ReplicaSystemFactory; import org.apache.accumulo.core.conf.Property; @@ -49,6 +49,8 @@ import org.apache.accumulo.core.replication.ReplicationTarget; import org.apache.accumulo.core.replication.StatusUtil; import org.apache.accumulo.core.replication.proto.Replication.Status; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.server.AccumuloServerContext; +import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.zookeeper.ZooCache; import org.apache.hadoop.io.Text; import org.slf4j.Logger; @@ -61,50 +63,44 @@ import com.google.protobuf.InvalidProtocolBufferException; public class ReplicationUtil { private static final Logger log = LoggerFactory.getLogger(ReplicationUtil.class); - private static final String REPLICATION_TARGET_PREFIX = Property.TABLE_REPLICATION_TARGET.getKey(); - private ZooCache zooCache; - public ReplicationUtil() { - this(new ZooCache()); + private final AccumuloServerContext context; + + public ReplicationUtil(AccumuloServerContext context) { + this(context, new ZooCache()); } - public ReplicationUtil(ZooCache cache) { + public ReplicationUtil(AccumuloServerContext context, ZooCache cache) { this.zooCache = cache; + this.context = context; } - public int getMaxReplicationThreads(Map systemProperties, MasterMonitorInfo mmi) { + public int getMaxReplicationThreads(MasterMonitorInfo mmi) { int activeTservers = mmi.getTServerInfoSize(); // The number of threads each tserver will use at most to replicate data - int replicationThreadsPerServer = Integer.parseInt(systemProperties.get(Property.REPLICATION_WORKER_THREADS.getKey())); + int replicationThreadsPerServer = Integer.parseInt(context.getConfiguration().get(Property.REPLICATION_WORKER_THREADS)); // The total number of "slots" we have to replicate data return activeTservers * replicationThreadsPerServer; } - public int getMaxReplicationThreads(Connector conn, MasterMonitorInfo mmi) throws AccumuloException, AccumuloSecurityException { - return getMaxReplicationThreads(conn.instanceOperations().getSystemConfiguration(), mmi); - } - /** * Extract replication peers from system configuration * - * @param systemProperties - * System properties, typically from Connector.instanceOperations().getSystemConfiguration() * @return Configured replication peers */ - public Map getPeers(Map systemProperties) { + public Map getPeers() { Map peers = new HashMap<>(); - String definedPeersPrefix = Property.REPLICATION_PEERS.getKey(); // Get the defined peers and what ReplicaSystem impl they're using - for (Entry property : systemProperties.entrySet()) { + for (Entry property : context.getConfiguration().getAllPropertiesWithPrefix(Property.REPLICATION_PEERS).entrySet()) { String key = property.getKey(); // Filter out cruft that we don't want - if (key.startsWith(definedPeersPrefix) && !key.startsWith(Property.REPLICATION_PEER_USER.getKey()) + if (!key.startsWith(Property.REPLICATION_PEER_USER.getKey()) && !key.startsWith(Property.REPLICATION_PEER_PASSWORD.getKey())) { - String peerName = property.getKey().substring(definedPeersPrefix.length()); + String peerName = property.getKey().substring(Property.REPLICATION_PEERS.getKey().length()); ReplicaSystem replica; try { replica = ReplicaSystemFactory.get(property.getValue()); @@ -120,54 +116,43 @@ public class ReplicationUtil { return peers; } - public Set getReplicationTargets(TableOperations tops) { + public Set getReplicationTargets() { // The total set of configured targets final Set allConfiguredTargets = new HashSet<>(); - final Map tableNameToId = tops.tableIdMap(); + final Map tableNameToId = Tables.getNameToIdMap(context.getInstance()); - for (String table : tops.list()) { + for (String table : tableNameToId.keySet()) { if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) { continue; } + String localId = tableNameToId.get(table); if (null == localId) { log.trace("Could not determine ID for {}", table); continue; } - Iterable> propertiesForTable; - try { - propertiesForTable = tops.getProperties(table); - } catch (AccumuloException e) { - log.debug("Could not fetch properties for " + table, e); - continue; - } catch (TableNotFoundException e) { - log.debug("Could not fetch properties for " + table, e); - continue; - } - - for (Entry prop : propertiesForTable) { - if (prop.getKey().startsWith(REPLICATION_TARGET_PREFIX)) { - String peerName = prop.getKey().substring(REPLICATION_TARGET_PREFIX.length()); + TableConfiguration tableConf = context.getServerConfigurationFactory().getTableConfiguration(localId); + for (Entry prop : tableConf.getAllPropertiesWithPrefix(Property.TABLE_REPLICATION_TARGET).entrySet()) { + String peerName = prop.getKey().substring(Property.TABLE_REPLICATION_TARGET.getKey().length()); String remoteIdentifier = prop.getValue(); ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId); allConfiguredTargets.add(target); } } - } return allConfiguredTargets; } - public Map getPendingReplications(Connector conn) { + public Map getPendingReplications() { final Map counts = new HashMap<>(); // Read over the queued work BatchScanner bs; try { - bs = conn.createBatchScanner(ReplicationTable.NAME, Authorizations.EMPTY, 4); - } catch (TableNotFoundException e) { + bs = context.getConnector().createBatchScanner(ReplicationTable.NAME, Authorizations.EMPTY, 4); + } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) { log.debug("No replication table exists", e); return counts; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java index 8049003..e473822 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java @@ -43,7 +43,7 @@ import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.ByteBufferUtil; -import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.security.handler.Authenticator; import org.apache.accumulo.server.security.handler.Authorizor; import org.apache.accumulo.server.security.handler.PermissionHandler; @@ -60,26 +60,26 @@ public class AuditedSecurityOperation extends SecurityOperation { public static final String AUDITLOG = "Audit"; public static final Logger audit = Logger.getLogger(AUDITLOG); - public AuditedSecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) { - super(author, authent, pm, instanceId); + public AuditedSecurityOperation(AccumuloServerContext context, Authorizor author, Authenticator authent, PermissionHandler pm) { + super(context, author, authent, pm); } - public static synchronized SecurityOperation getInstance() { - String instanceId = HdfsZooInstance.getInstance().getInstanceID(); - return getInstance(instanceId, false); + public static synchronized SecurityOperation getInstance(AccumuloServerContext context) { + return getInstance(context, false); } - public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) { + public static synchronized SecurityOperation getInstance(AccumuloServerContext context, boolean initialize) { if (instance == null) { - instance = new AuditedSecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, - initialize), instanceId); + String instanceId = context.getInstance().getInstanceID(); + instance = new AuditedSecurityOperation(context, getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler( + instanceId, initialize)); } return instance; } - private static String getTableName(String tableId) { + private String getTableName(String tableId) { try { - return Tables.getTableName(HdfsZooInstance.getInstance(), tableId); + return Tables.getTableName(context.getInstance(), tableId); } catch (TableNotFoundException e) { return "Unknown Table with ID " + tableId; } @@ -93,13 +93,13 @@ public class AuditedSecurityOperation extends SecurityOperation { return auths; } - private static boolean shouldAudit(TCredentials credentials, String tableId) { + private boolean shouldAudit(TCredentials credentials, String tableId) { return (audit.isInfoEnabled() || audit.isEnabledFor(Level.WARN)) && !tableId.equals(MetadataTable.ID) && shouldAudit(credentials); } // Is INFO the right level to check? Do we even need that check? - private static boolean shouldAudit(TCredentials credentials) { - return !SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName()); + private boolean shouldAudit(TCredentials credentials) { + return !context.getCredentials().getToken().getClass().getName().equals(credentials.getTokenClassName()); } /* http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java index 71dcdcf..5e81018 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java @@ -48,7 +48,7 @@ import org.apache.accumulo.core.security.NamespacePermission; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.security.thrift.TCredentials; -import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.security.handler.Authenticator; import org.apache.accumulo.server.security.handler.Authorizor; import org.apache.accumulo.server.security.handler.PermissionHandler; @@ -72,17 +72,19 @@ public class SecurityOperation { private final ZooCache zooCache; private final String ZKUserPath; + protected final AccumuloServerContext context; + static SecurityOperation instance; - public static synchronized SecurityOperation getInstance() { - String instanceId = HdfsZooInstance.getInstance().getInstanceID(); - return getInstance(instanceId, false); + public static synchronized SecurityOperation getInstance(AccumuloServerContext context) { + return getInstance(context, false); } - public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) { + public static synchronized SecurityOperation getInstance(AccumuloServerContext context, boolean initialize) { if (instance == null) { - instance = new SecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, initialize), - instanceId); + String instanceId = context.getInstance().getInstanceID(); + instance = new SecurityOperation(context, getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, + initialize)); } return instance; } @@ -108,13 +110,14 @@ public class SecurityOperation { return toRet; } - protected SecurityOperation(String instanceId) { - ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users"; + protected SecurityOperation(AccumuloServerContext context) { + this.context = context; + ZKUserPath = Constants.ZROOT + "/" + context.getInstance().getInstanceID() + "/users"; zooCache = new ZooCache(); } - public SecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) { - this(instanceId); + public SecurityOperation(AccumuloServerContext context, Authorizor author, Authenticator authent, PermissionHandler pm) { + this(context); authorizor = author; authenticator = authent; permHandle = pm; @@ -149,16 +152,16 @@ public class SecurityOperation { } public boolean isSystemUser(TCredentials credentials) { - return SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName()); + return context.getCredentials().getToken().getClass().getName().equals(credentials.getTokenClassName()); } protected void authenticate(TCredentials credentials) throws ThriftSecurityException { - if (!credentials.getInstanceId().equals(HdfsZooInstance.getInstance().getInstanceID())) + if (!credentials.getInstanceId().equals(context.getInstance().getInstanceID())) throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.INVALID_INSTANCEID); Credentials creds = Credentials.fromThrift(credentials); if (isSystemUser(credentials)) { - if (!(SystemCredentials.get().equals(creds))) { + if (!(context.getCredentials().equals(creds))) { throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS); } } else { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java index 33a5158..a59d57c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java @@ -36,7 +36,6 @@ import org.apache.accumulo.core.security.Credentials; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.Base64; import org.apache.accumulo.server.ServerConstants; -import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.hadoop.io.Writable; /** @@ -48,7 +47,6 @@ public final class SystemCredentials extends Credentials { private static final SecurityPermission SYSTEM_CREDENTIALS_PERMISSION = new SecurityPermission("systemCredentialsPermission"); - private static SystemCredentials SYSTEM_CREDS = null; private static final String SYSTEM_PRINCIPAL = "!SYSTEM"; private final TCredentials AS_THRIFT; @@ -58,14 +56,6 @@ public final class SystemCredentials extends Credentials { AS_THRIFT = super.toThrift(instance); } - public static SystemCredentials get() { - check_permission(); - if (SYSTEM_CREDS == null) { - SYSTEM_CREDS = new SystemCredentials(HdfsZooInstance.getInstance()); - } - return SYSTEM_CREDS; - } - private static void check_permission() { SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -75,12 +65,6 @@ public final class SystemCredentials extends Credentials { public static SystemCredentials get(Instance instance) { check_permission(); - /* Special case to avoid duplicating SYSTEM_CREDS */ - if (null != SYSTEM_CREDS) { - if (SYSTEM_CREDS.AS_THRIFT.getInstanceId().equals(instance.getInstanceID())) { - return SYSTEM_CREDS; - } - } return new SystemCredentials(instance); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java index 254b62c..651d31d 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java @@ -36,9 +36,9 @@ import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.InstanceOperations; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.client.impl.ClientExec; import org.apache.accumulo.core.client.impl.MasterClient; -import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; @@ -49,9 +49,9 @@ import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.trace.Tracer; import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.cli.ClientOpts; import org.apache.accumulo.server.conf.ServerConfigurationFactory; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -169,17 +169,20 @@ public class Admin { return; } Instance instance = opts.getInstance(); - AccumuloConfiguration conf = new ServerConfigurationFactory(instance).getConfiguration(); + ServerConfigurationFactory confFactory = new ServerConfigurationFactory(instance); try { - String principal; - AuthenticationToken token; + ClientContext context; if (opts.getToken() == null) { - principal = SystemCredentials.get().getPrincipal(); - token = SystemCredentials.get().getToken(); + context = new AccumuloServerContext(confFactory); } else { - principal = opts.principal; - token = opts.getToken(); + final Credentials userCreds = new Credentials(opts.principal, opts.getToken()); + context = new AccumuloServerContext(confFactory) { + @Override + public synchronized Credentials getCredentials() { + return userCreds; + } + }; } int rc = 0; @@ -187,36 +190,36 @@ public class Admin { if (cl.getParsedCommand().equals("listInstances")) { ListInstances.listInstances(instance.getZooKeepers(), listIntancesOpts.printAll, listIntancesOpts.printErrors); } else if (cl.getParsedCommand().equals("ping")) { - if (ping(instance, principal, token, pingCommand.args) != 0) + if (ping(context, pingCommand.args) != 0) rc = 4; } else if (cl.getParsedCommand().equals("checkTablets")) { System.out.println("\n*** Looking for offline tablets ***\n"); - if (FindOfflineTablets.findOffline(instance, new Credentials(principal, token), checkTabletsCommand.table) != 0) + if (FindOfflineTablets.findOffline(context, checkTabletsCommand.table) != 0) rc = 5; System.out.println("\n*** Looking for missing files ***\n"); if (checkTabletsCommand.table == null) { - if (RemoveEntriesForMissingFiles.checkAllTables(instance, principal, token, checkTabletsCommand.fixFiles) != 0) + if (RemoveEntriesForMissingFiles.checkAllTables(context, checkTabletsCommand.fixFiles) != 0) rc = 6; } else { - if (RemoveEntriesForMissingFiles.checkTable(instance, principal, token, checkTabletsCommand.table, checkTabletsCommand.fixFiles) != 0) + if (RemoveEntriesForMissingFiles.checkTable(context, checkTabletsCommand.table, checkTabletsCommand.fixFiles) != 0) rc = 6; } } else if (cl.getParsedCommand().equals("stop")) { - stopTabletServer(conf, instance, new Credentials(principal, token), stopOpts.args, opts.force); + stopTabletServer(context, stopOpts.args, opts.force); } else if (cl.getParsedCommand().equals("dumpConfig")) { - printConfig(instance, principal, token, dumpConfigCommand); + printConfig(context, dumpConfigCommand); } else if (cl.getParsedCommand().equals("volumes")) { - ListVolumesUsed.listVolumes(instance, principal, token); + ListVolumesUsed.listVolumes(context); } else if (cl.getParsedCommand().equals("randomizeVolumes")) { - rc = RandomizeVolumes.randomize(instance.getConnector(principal, token), randomizeVolumesOpts.table); + rc = RandomizeVolumes.randomize(context.getConnector(), randomizeVolumesOpts.table); } else { everything = cl.getParsedCommand().equals("stopAll"); if (everything) - flushAll(instance, principal, token); + flushAll(context); - stopServer(instance, new Credentials(principal, token), everything); + stopServer(context, everything); } if (rc != 0) @@ -233,10 +236,9 @@ public class Admin { } } - private static int ping(Instance instance, String principal, AuthenticationToken token, List args) throws AccumuloException, - AccumuloSecurityException { + private static int ping(ClientContext context, List args) throws AccumuloException, AccumuloSecurityException { - InstanceOperations io = instance.getConnector(principal, token).instanceOperations(); + InstanceOperations io = context.getConnector().instanceOperations(); if (args.size() == 0) { args = io.getTabletServers(); @@ -263,8 +265,7 @@ public class Admin { * it takes too long. * */ - private static void flushAll(final Instance instance, final String principal, final AuthenticationToken token) throws AccumuloException, - AccumuloSecurityException { + private static void flushAll(final ClientContext context) throws AccumuloException, AccumuloSecurityException { final AtomicInteger flushesStarted = new AtomicInteger(0); @@ -273,7 +274,7 @@ public class Admin { @Override public void run() { try { - Connector conn = instance.getConnector(principal, token); + Connector conn = context.getConnector(); Set tables = conn.tableOperations().tableIdMap().keySet(); for (String table : tables) { if (table.equals(MetadataTable.NAME)) @@ -311,30 +312,29 @@ public class Admin { } } - private static void stopServer(final Instance instance, final Credentials credentials, final boolean tabletServersToo) throws AccumuloException, - AccumuloSecurityException { - MasterClient.execute(instance, new ClientExec() { + private static void stopServer(final ClientContext context, final boolean tabletServersToo) throws AccumuloException, AccumuloSecurityException { + MasterClient.execute(context, new ClientExec() { @Override public void execute(MasterClientService.Client client) throws Exception { - client.shutdown(Tracer.traceInfo(), credentials.toThrift(instance), tabletServersToo); + client.shutdown(Tracer.traceInfo(), context.rpcCreds(), tabletServersToo); } }); } - private static void stopTabletServer(final AccumuloConfiguration conf, final Instance instance, final Credentials creds, List servers, final boolean force) throws AccumuloException, + private static void stopTabletServer(final ClientContext context, List servers, final boolean force) throws AccumuloException, AccumuloSecurityException { - if (instance.getMasterLocations().size() == 0) { + if (context.getInstance().getMasterLocations().size() == 0) { log.info("No masters running. Not attempting safe unload of tserver."); return; } for (String server : servers) { - HostAndPort address = AddressUtil.parseAddress(server, conf.getPort(Property.TSERV_CLIENTPORT)); + HostAndPort address = AddressUtil.parseAddress(server, context.getConfiguration().getPort(Property.TSERV_CLIENTPORT)); final String finalServer = address.toString(); log.info("Stopping server " + finalServer); - MasterClient.execute(instance, new ClientExec() { + MasterClient.execute(context, new ClientExec() { @Override public void execute(MasterClientService.Client client) throws Exception { - client.shutdownTabletServer(Tracer.traceInfo(), creds.toThrift(instance), finalServer, force); + client.shutdownTabletServer(Tracer.traceInfo(), context.rpcCreds(), finalServer, force); } }); } @@ -350,7 +350,7 @@ public class Admin { private static Map siteConfig, systemConfig; private static List localUsers; - public static void printConfig(Instance instance, String principal, AuthenticationToken token, DumpConfigCommand opts) throws Exception { + public static void printConfig(ClientContext context, DumpConfigCommand opts) throws Exception { File outputDirectory = null; if (opts.directory != null) { @@ -362,7 +362,7 @@ public class Admin { throw new IllegalArgumentException(opts.directory + " is not writable"); } } - Connector connector = instance.getConnector(principal, token); + Connector connector = context.getConnector(); defaultConfig = AccumuloConfiguration.getDefaultConfiguration(); siteConfig = connector.instanceOperations().getSiteConfiguration(); systemConfig = connector.instanceOperations().getSystemConfiguration(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java index 721d4e2..e557268 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java @@ -21,18 +21,18 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.client.impl.Tables; -import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.master.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; -import org.apache.accumulo.core.security.Credentials; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.cli.ClientOpts; +import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.master.LiveTServerSet; import org.apache.accumulo.server.master.LiveTServerSet.Listener; import org.apache.accumulo.server.master.state.DistributedStoreException; @@ -41,7 +41,6 @@ import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletLocationState; import org.apache.accumulo.server.master.state.TabletState; import org.apache.accumulo.server.master.state.ZooTabletStateStore; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.tables.TableManager; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; @@ -52,17 +51,15 @@ public class FindOfflineTablets { public static void main(String[] args) throws Exception { ClientOpts opts = new ClientOpts(); opts.parseArgs(FindOfflineTablets.class.getName(), args); - Instance instance = opts.getInstance(); - SystemCredentials creds = SystemCredentials.get(); - - findOffline(instance, creds, null); + AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(opts.getInstance())); + findOffline(context, null); } - static int findOffline(Instance instance, Credentials creds, String tableName) throws AccumuloException, TableNotFoundException { + static int findOffline(ClientContext context, String tableName) throws AccumuloException, TableNotFoundException { final AtomicBoolean scanning = new AtomicBoolean(false); - LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() { + LiveTServerSet tservers = new LiveTServerSet(context, new Listener() { @Override public void update(LiveTServerSet current, Set deleted, Set added) { if (!deleted.isEmpty() && scanning.get()) @@ -91,7 +88,7 @@ public class FindOfflineTablets { return 0; System.out.println("Scanning " + RootTable.NAME); - Iterator rootScanner = new MetaDataTableScanner(instance, creds, MetadataSchema.TabletsSection.getRange(), RootTable.NAME); + Iterator rootScanner = new MetaDataTableScanner(context, MetadataSchema.TabletsSection.getRange(), RootTable.NAME); if ((offline = checkTablets(rootScanner, tservers)) > 0) return offline; @@ -102,11 +99,11 @@ public class FindOfflineTablets { Range range = MetadataSchema.TabletsSection.getRange(); if (tableName != null) { - String tableId = Tables.getTableId(instance, tableName); + String tableId = Tables.getTableId(context.getInstance(), tableName); range = new KeyExtent(new Text(tableId), null, null).toMetadataRange(); } - MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, creds, range, MetadataTable.NAME); + MetaDataTableScanner metaScanner = new MetaDataTableScanner(context, range, MetadataTable.NAME); try { return checkTablets(metaScanner, tservers); } finally { http://git-wip-us.apache.org/repos/asf/accumulo/blob/42c25faa/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java index b876392..b288aac 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java @@ -21,9 +21,8 @@ import java.util.Map.Entry; import java.util.TreeSet; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; @@ -31,9 +30,10 @@ import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.AccumuloServerContext; import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.fs.VolumeManager.FileType; -import org.apache.accumulo.server.security.SystemCredentials; import org.apache.hadoop.fs.Path; /** @@ -41,10 +41,8 @@ import org.apache.hadoop.fs.Path; */ public class ListVolumesUsed { - - public static void main(String[] args) throws Exception { - listVolumes(HdfsZooInstance.getInstance(), SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()); + listVolumes(new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()))); } private static String getTableURI(String rootTabletDir) { @@ -84,8 +82,6 @@ public class ListVolumesUsed { } - - private static void listTable(String name, Connector conn) throws Exception { System.out.println("Listing volumes referenced in " + name + " tablets section"); @@ -129,8 +125,8 @@ public class ListVolumesUsed { System.out.println("\tVolume : " + volume); } - public static void listVolumes(Instance instance, String principal, AuthenticationToken token) throws Exception { - Connector conn = instance.getConnector(principal, token); + public static void listVolumes(ClientContext context) throws Exception { + Connector conn = context.getConnector(); listZookeeper(); System.out.println(); listTable(RootTable.NAME, conn);