accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [60/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Date Wed, 09 Apr 2014 17:58:31 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index cdcdfba,0000000..98d3c73
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,276 -1,0 +1,272 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.metrics;
 +
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.Writer;
 +import java.lang.management.ManagementFactory;
 +import java.text.SimpleDateFormat;
 +import java.util.Date;
 +import java.util.concurrent.ConcurrentHashMap;
 +
 +import javax.management.MBeanServer;
 +import javax.management.ObjectName;
 +import javax.management.StandardMBean;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.commons.lang.builder.ToStringBuilder;
 +import org.apache.commons.lang.time.DateUtils;
 +
 +public abstract class AbstractMetricsImpl {
 +  
 +  public class Metric {
 +    
 +    private long count = 0;
 +    private long avg = 0;
 +    private long min = 0;
 +    private long max = 0;
 +    
 +    public long getCount() {
 +      return count;
 +    }
 +    
 +    public long getAvg() {
 +      return avg;
 +    }
 +    
 +    public long getMin() {
 +      return min;
 +    }
 +    
 +    public long getMax() {
 +      return max;
 +    }
 +    
 +    public void incCount() {
 +      count++;
 +    }
 +    
 +    public void addAvg(long a) {
 +      if (a < 0)
 +        return;
 +      avg = (long) ((avg * .8) + (a * .2));
 +    }
 +    
 +    public void addMin(long a) {
 +      if (a < 0)
 +        return;
 +      min = Math.min(min, a);
 +    }
 +    
 +    public void addMax(long a) {
 +      if (a < 0)
 +        return;
 +      max = Math.max(max, a);
 +    }
 +    
 +    @Override
 +    public String toString() {
 +      return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
 +    }
 +    
 +  }
 +  
 +  static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
 +  
 +  private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
 +  
 +  private boolean currentlyLogging = false;
 +  
 +  private File logDir = null;
 +  
 +  private String metricsPrefix = null;
 +  
 +  private Date today = new Date();
 +  
 +  private File logFile = null;
 +  
 +  private Writer logWriter = null;
 +  
 +  private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
 +  
 +  private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
 +  
 +  private MetricsConfiguration config = null;
 +  
 +  public AbstractMetricsImpl() {
 +    this.metricsPrefix = getMetricsPrefix();
 +    config = new MetricsConfiguration(metricsPrefix);
 +  }
 +  
 +  /**
 +   * Registers a StandardMBean with the MBean Server
-    * 
-    * @throws Exception
 +   */
 +  public void register(StandardMBean mbean) throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(mbean, getObjectName());
 +    
 +    setupLogging();
 +  }
 +  
 +  /**
 +   * Registers this MBean with the MBean Server
-    * 
-    * @throws Exception
 +   */
 +  public void register() throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(this, getObjectName());
 +    setupLogging();
 +  }
 +  
 +  public void createMetric(String name) {
 +    registry.put(name, new Metric());
 +  }
 +  
 +  public Metric getMetric(String name) {
 +    return registry.get(name);
 +  }
 +  
 +  public long getMetricCount(String name) {
 +    return registry.get(name).getCount();
 +  }
 +  
 +  public long getMetricAvg(String name) {
 +    return registry.get(name).getAvg();
 +  }
 +  
 +  public long getMetricMin(String name) {
 +    return registry.get(name).getMin();
 +  }
 +  
 +  public long getMetricMax(String name) {
 +    return registry.get(name).getMax();
 +  }
 +  
 +  private void setupLogging() throws IOException {
 +    if (null == config.getMetricsConfiguration())
 +      return;
 +    // If we are already logging, then return
 +    if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +      // Check to see if directory exists, else make it
 +      String mDir = config.getMetricsConfiguration().getString("logging.dir");
 +      if (null != mDir) {
 +        File dir = new File(mDir);
 +        if (!dir.isDirectory())
 +          if (!dir.mkdir()) 
 +            log.warn("Could not create log directory: " + dir);
 +        logDir = dir;
 +        // Create new log file
 +        startNewLog();
 +      }
 +      currentlyLogging = true;
 +    }
 +  }
 +  
 +  private void startNewLog() throws IOException {
 +    if (null != logWriter) {
 +      logWriter.flush();
 +      logWriter.close();
 +    }
 +    logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
 +    if (!logFile.exists()) {
 +      if (!logFile.createNewFile()) {
 +        log.error("Unable to create new log file");
 +        currentlyLogging = false;
 +        return;
 +      }
 +    }
 +    logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
 +  }
 +  
 +  private void writeToLog(String name) throws IOException {
 +    if (null == logWriter)
 +      return;
 +    // Increment the date if we have to
 +    Date now = new Date();
 +    if (!DateUtils.isSameDay(today, now)) {
 +      today = now;
 +      startNewLog();
 +    }
 +    logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
 +  }
 +  
 +  public void add(String name, long time) {
 +    if (isEnabled()) {
 +      registry.get(name).incCount();
 +      registry.get(name).addAvg(time);
 +      registry.get(name).addMin(time);
 +      registry.get(name).addMax(time);
 +      // If we are not currently logging and should be, then initialize
 +      if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        try {
 +          setupLogging();
 +        } catch (IOException ioe) {
 +          log.error("Error setting up log", ioe);
 +        }
 +      } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        // if we are currently logging and shouldn't be, then close logs
 +        try {
 +          logWriter.flush();
 +          logWriter.close();
 +          logWriter = null;
 +          logFile = null;
 +        } catch (Exception e) {
 +          log.error("Error stopping metrics logging", e);
 +        }
 +        currentlyLogging = false;
 +      }
 +      if (currentlyLogging) {
 +        try {
 +          writeToLog(name);
 +        } catch (IOException ioe) {
 +          log.error("Error writing to metrics log", ioe);
 +        }
 +      }
 +    }
 +  }
 +  
 +  public boolean isEnabled() {
 +    return config.isEnabled();
 +  }
 +  
 +  protected abstract ObjectName getObjectName();
 +  
 +  protected abstract String getMetricsPrefix();
 +  
 +  @Override
 +  protected void finalize() {
 +    if (null != logWriter) {
 +      try {
 +        logWriter.close();
 +      } catch (Exception e) {
 +        // do nothing
 +      } finally {
 +        logWriter = null;
 +      }
 +    }
 +    logFile = null;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 34d43f2,0000000..bbaf592
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,175 -1,0 +1,171 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
 +import java.nio.ByteBuffer;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class ZKAuthorizor implements Authorizor {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static Authorizor zkAuthorizorInstance = null;
 +
 +  private final String ZKUserAuths = "/Authorizations";
 +
 +  private String ZKUserPath;
 +  private final ZooCache zooCache;
 +
 +  public static synchronized Authorizor getInstance() {
 +    if (zkAuthorizorInstance == null)
 +      zkAuthorizorInstance = new ZKAuthorizor();
 +    return zkAuthorizorInstance;
 +  }
 +
 +  public ZKAuthorizor() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +  }
 +
 +  @Override
 +  public Authorizations getCachedUserAuthorizations(String user) {
 +    byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
 +    if (authsBytes != null)
 +      return ZKSecurityTool.convertAuthorizations(authsBytes);
 +    return Authorizations.EMPTY;
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the metadata tables
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
 +        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
-   /**
-    * @param user
-    * @throws AccumuloSecurityException
-    */
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void dropUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
 +            NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
 +    Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
 +    for (ByteBuffer auth : auths)
 +      if (!userauths.contains(auth))
 +        return false;
 +    return true;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 6319653,0000000..1b7e7d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,517 -1,0 +1,513 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.Code;
 +
 +/**
 + * 
 + */
 +public class ZKPermHandler implements PermissionHandler {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static PermissionHandler zkPermHandlerInstance = null;
 +
 +  private String ZKUserPath;
 +  private String ZKTablePath;
 +  private String ZKNamespacePath;
 +  private final ZooCache zooCache;
 +  private final String ZKUserSysPerms = "/System";
 +  private final String ZKUserTablePerms = "/Tables";
 +  private final String ZKUserNamespacePerms = "/Namespaces";
 +
 +  public static synchronized PermissionHandler getInstance() {
 +    if (zkPermHandlerInstance == null)
 +      zkPermHandlerInstance = new ZKPermHandler();
 +    return zkPermHandlerInstance;
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +    ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
 +    ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
 +  }
 +
 +  public ZKPermHandler() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
 +    byte[] serializedPerms;
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
 +      ZooReaderWriter.getRetryingInstance().sync(path);
 +      serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the table was just deleted?
 +        try {
 +          // check for existence:
 +          ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new TableNotFoundException(null, table, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
 +    byte[] serializedPerms;
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
 +      ZooReaderWriter.getRetryingInstance().sync(path);
 +      serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the namespace was just deleted?
 +        try {
 +          // check for existence:
 +          ZooReaderWriter.getRetryingInstance().getData(ZKNamespacePath + "/" + namespace, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
 +      NamespaceNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    try {
 +      byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +      Set<SystemPermission> perms;
 +      if (permBytes == null) {
 +        perms = new TreeSet<SystemPermission>();
 +      } else {
 +        perms = ZKSecurityTool.convertSystemPermissions(permBytes);
 +      }
 +
 +      if (perms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    Set<TablePermission> tablePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null)
 +      tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    else
 +      tablePerms = new TreeSet<TablePermission>();
 +
 +    try {
 +      if (tablePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +          IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    Set<NamespacePermission> namespacePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null)
 +      namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    else
 +      namespacePerms = new TreeSet<NamespacePermission>();
 +
 +    try {
 +      if (namespacePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +          IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +
 +    // User had no system permission, nothing to revoke.
 +    if (sysPermBytes == null)
 +      return;
 +
 +    Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
 +
 +    try {
 +      if (sysPerms.remove(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +
 +    // User had no table permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    try {
 +      if (tablePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        if (tablePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +
 +    // User had no namespace permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    try {
 +      if (namespacePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        if (namespacePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanTablePermissions(String table) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the system tables
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
 +    Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
 +        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
 +      for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
 +        createTablePerm(rootuser, entry.getKey(), entry.getValue());
 +      for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
 +        createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
-   /**
-    * @param user
-    * @throws AccumuloSecurityException
-    */
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
 +          ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
 +          ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms;
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserSysPerms;
 +      ZooReaderWriter.getRetryingInstance().sync(path);
 +      perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initTable(String table) throws AccumuloSecurityException {
 +    // All proper housekeeping is done on delete and permission granting, no work needs to be done here
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
 +import java.util.ArrayList;
 +import java.util.List;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.security.handler.Authenticator;
 +import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 +
 +/**
 + * 
 + */
 +public class LoginProperties {
 +  
-   /**
-    * @param args
-    */
 +  public static void main(String[] args) throws Exception {
 +    AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
 +    Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
 +        .asSubclass(Authenticator.class).newInstance();
 +    
 +    List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
 +    
 +    for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
 +      tokenProps.add(tokenType.newInstance().getProperties());
 +    }
 +    
 +    System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
 +    for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
 +      System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
 +      
 +      for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
 +        System.out.println("\t\t" + tokenProperty);
 +      }
 +      
 +      System.out.println();
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 6e5607e,0000000..37ef5f1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,128 -1,0 +1,124 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
 +import java.io.FileInputStream;
 +import java.io.InputStream;
 +import java.util.Stack;
 +
 +import javax.xml.parsers.SAXParser;
 +import javax.xml.parsers.SAXParserFactory;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.commons.codec.binary.Base64;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.xml.sax.Attributes;
 +import org.xml.sax.SAXException;
 +import org.xml.sax.helpers.DefaultHandler;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class RestoreZookeeper {
 +  
 +  private static class Restore extends DefaultHandler {
 +    IZooReaderWriter zk = null;
 +    Stack<String> cwd = new Stack<String>();
 +    boolean overwrite = false;
 +    
 +    Restore(IZooReaderWriter zk, boolean overwrite) {
 +      this.zk = zk;
 +      this.overwrite = overwrite;
 +    }
 +    
 +    @Override
 +    public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
 +      if ("node".equals(name)) {
 +        String child = attributes.getValue("name");
 +        if (child == null)
 +          throw new RuntimeException("name attribute not set");
 +        String encoding = attributes.getValue("encoding");
 +        String value = attributes.getValue("value");
 +        if (value == null)
 +          value = "";
 +        String path = cwd.lastElement() + "/" + child;
 +        create(path, value, encoding);
 +        cwd.push(path);
 +      } else if ("dump".equals(name)) {
 +        String root = attributes.getValue("root");
 +        if (root.equals("/"))
 +          cwd.push("");
 +        else
 +          cwd.push(root);
 +        create(root, "", Constants.UTF8.name());
 +      }
 +    }
 +    
 +    @Override
 +    public void endElement(String uri, String localName, String name) throws SAXException {
 +      cwd.pop();
 +    }
 +    
 +    // assume UTF-8 if not "base64"
 +    private void create(String path, String value, String encoding) {
 +      byte[] data = value.getBytes(Constants.UTF8);
 +      if ("base64".equals(encoding))
 +        data = Base64.decodeBase64(data);
 +      try {
 +        try {
 +          zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
 +        } catch (KeeperException e) {
 +          if (e.code().equals(KeeperException.Code.NODEEXISTS))
 +            throw new RuntimeException(path + " exists.  Remove it first.");
 +          throw e;
 +        }
 +      } catch (Exception e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = {"-z", "--keepers"})
 +    String keepers = "localhost:2181";
 +    @Parameter(names = "--overwrite")
 +    boolean overwrite = false;
 +    @Parameter(names = "--file")
 +    String file;
 +  }
 +  
-   /**
-    * @param args
-    * @throws Exception
-    */
 +  public static void main(String[] args) throws Exception {
 +    Logger.getRootLogger().setLevel(Level.WARN);
 +    Opts opts = new Opts();
 +    opts.parseArgs(RestoreZookeeper.class.getName(), args);
 +    
 +    InputStream in = System.in;
 +    if (opts.file != null) {
 +      in = new FileInputStream(opts.file);
 +    }
 +    
 +    SAXParserFactory factory = SAXParserFactory.newInstance();
 +    SAXParser parser = factory.newSAXParser();
 +    parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
 +    in.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index cb932d7,0000000..9a54927
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,290 -1,0 +1,287 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Comparator;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.NumUtil;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.server.cli.ClientOpts;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class TableDiskUsage {
 +
 +  private static final Logger log = Logger.getLogger(Logger.class);
 +  private int nextInternalId = 0;
 +  private Map<String,Integer> internalIds = new HashMap<String,Integer>();
 +  private Map<Integer,String> externalIds = new HashMap<Integer,String>();
 +  private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
 +  private Map<String,Long> fileSizes = new HashMap<String,Long>();
 +
 +  void addTable(String tableId) {
 +    if (internalIds.containsKey(tableId))
 +      throw new IllegalArgumentException("Already added table " + tableId);
 +
 +    int iid = nextInternalId++;
 +
 +    internalIds.put(tableId, iid);
 +    externalIds.put(iid, tableId);
 +  }
 +
 +  void linkFileAndTable(String tableId, String file) {
 +    int internalId = internalIds.get(tableId);
 +
 +    Integer[] tables = tableFiles.get(file);
 +    if (tables == null) {
 +      tables = new Integer[internalIds.size()];
 +      for (int i = 0; i < tables.length; i++)
 +        tables[i] = 0;
 +      tableFiles.put(file, tables);
 +    }
 +
 +    tables[internalId] = 1;
 +  }
 +
 +  void addFileSize(String file, long size) {
 +    fileSizes.put(file, size);
 +  }
 +
 +  Map<List<String>,Long> calculateUsage() {
 +
 +    Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
 +
 +    for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
 +      log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
 +      List<Integer> key = Arrays.asList(entry.getValue());
 +      Long size = fileSizes.get(entry.getKey());
 +
 +      Long tablesUsage = usage.get(key);
 +      if (tablesUsage == null)
 +        tablesUsage = 0l;
 +
 +      tablesUsage += size;
 +
 +      usage.put(key, tablesUsage);
 +
 +    }
 +
 +    Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
 +
 +    for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
 +      List<String> externalKey = new ArrayList<String>();
 +      List<Integer> key = entry.getKey();
 +      for (int i = 0; i < key.size(); i++)
 +        if (key.get(i) != 0)
 +          externalKey.add(externalIds.get(i));
 +
 +      externalUsage.put(externalKey, entry.getValue());
 +    }
 +
 +    return externalUsage;
 +  }
 +
 +  public interface Printer {
 +    void print(String line);
 +  }
 +
 +  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, boolean humanReadable)
 +      throws TableNotFoundException, IOException {
 +    printDiskUsage(acuConf, tables, fs, conn, new Printer() {
 +      @Override
 +      public void print(String line) {
 +        System.out.println(line);
 +      }
 +    }, humanReadable);
 +  }
 +
 +  public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
 +      throws IOException {
 +    TableDiskUsage tdu = new TableDiskUsage();
 +
 +    for (String tableId : tableIds)
 +      tdu.addTable(tableId);
 +
 +    HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
 +    HashSet<String> emptyTableIds = new HashSet<String>();
 +    HashSet<String> nameSpacesReferenced = new HashSet<String>();
 +
 +    for (String tableId : tableIds) {
 +      Scanner mdScanner = null;
 +      try {
 +        mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      } catch (TableNotFoundException e) {
 +        throw new RuntimeException(e);
 +      }
 +      mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +      mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +
 +      if (!mdScanner.iterator().hasNext()) {
 +        emptyTableIds.add(tableId);
 +      }
 +
 +      for (Entry<Key,Value> entry : mdScanner) {
 +        String file = entry.getKey().getColumnQualifier().toString();
 +        String parts[] = file.split("/");
 +        String uniqueName = parts[parts.length - 1];
 +        if (file.contains(":") || file.startsWith("../")) {
 +          String ref = parts[parts.length - 3];
 +          if (!ref.equals(tableId)) {
 +            tablesReferenced.add(ref);
 +          }
 +          if (file.contains(":") && parts.length > 3) {
 +            List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
 +            nameSpacesReferenced.add(StringUtil.join(base, "/"));
 +          }
 +        }
 +
 +        tdu.linkFileAndTable(tableId, uniqueName);
 +      }
 +    }
 +
 +    for (String tableId : tablesReferenced) {
 +      for (String tableDir : nameSpacesReferenced) {
 +        FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
 +        if (files != null) {
 +          for (FileStatus fileStatus : files) {
 +            // Assumes that all filenames are unique
 +            String name = fileStatus.getPath().getName();
 +            tdu.addFileSize(name, fileStatus.getLen());
 +          }
 +        }
 +      }
 +    }
 +
 +    HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
 +    for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
 +      reverseTableIdMap.put(entry.getValue(), entry.getKey());
 +
 +    TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
 +
 +      @Override
 +      public int compare(TreeSet<String> o1, TreeSet<String> o2) {
 +        int len1 = o1.size();
 +        int len2 = o2.size();
 +
 +        int min = Math.min(len1, len2);
 +
 +        Iterator<String> iter1 = o1.iterator();
 +        Iterator<String> iter2 = o2.iterator();
 +
 +        int count = 0;
 +
 +        while (count < min) {
 +          String s1 = iter1.next();
 +          String s2 = iter2.next();
 +
 +          int cmp = s1.compareTo(s2);
 +
 +          if (cmp != 0)
 +            return cmp;
 +
 +          count++;
 +        }
 +
 +        return len1 - len2;
 +      }
 +    });
 +
 +    for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
 +      TreeSet<String> tableNames = new TreeSet<String>();
 +      for (String tableId : entry.getKey())
 +        tableNames.add(reverseTableIdMap.get(tableId));
 +
 +      usage.put(tableNames, entry.getValue());
 +    }
 +
 +    if (!emptyTableIds.isEmpty()) {
 +      TreeSet<String> emptyTables = new TreeSet<String>();
 +      for (String tableId : emptyTableIds) {
 +        emptyTables.add(reverseTableIdMap.get(tableId));
 +      }
 +      usage.put(emptyTables, 0L);
 +    }
 +
 +    return usage;
 +  }
 +
 +  public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer,
 +      boolean humanReadable) throws TableNotFoundException, IOException {
 +
 +    HashSet<String> tableIds = new HashSet<String>();
 +
 +    for (String tableName : tables) {
 +      String tableId = conn.tableOperations().tableIdMap().get(tableName);
 +      if (tableId == null)
 +        throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
 +
 +      tableIds.add(tableId);
 +    }
 +
 +    Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
 +
 +    String valueFormat = humanReadable ? "%9s" : "%,24d";
 +    for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
 +      Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
 +      printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
 +    }
 +  }
 +
 +  static class Opts extends ClientOpts {
 +    @Parameter(description = " <table> { <table> ... } ")
 +    List<String> tables = new ArrayList<String>();
 +  }
 +
-   /**
-    * @param args
-    */
 +  public static void main(String[] args) throws Exception {
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Opts opts = new Opts();
 +    opts.parseArgs(TableDiskUsage.class.getName(), args);
 +    Connector conn = opts.getConnector();
 +    org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class TabletServerLocks {
 +  
 +  static class Opts extends Help {
 +    @Parameter(names="-list")
 +    boolean list = false;
 +    @Parameter(names="-delete")
 +    String delete = null;
 +  }
-   /**
-    * @param args
-    */
 +  public static void main(String[] args) throws Exception {
 +    
 +    Instance instance = HdfsZooInstance.getInstance();
 +    String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +    Opts opts = new Opts();
 +    opts.parseArgs(TabletServerLocks.class.getName(), args);
 +    
 +    ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    
 +    if (opts.list) {
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      
 +      List<String> tabletServers = zoo.getChildren(tserverPath);
 +      
 +      for (String tabletServer : tabletServers) {
 +        byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
 +        String holder = null;
 +        if (lockData != null) {
 +          holder = new String(lockData, Constants.UTF8);
 +        }
 +        
 +        System.out.printf("%32s %16s%n", tabletServer, holder);
 +      }
 +    } else if (opts.delete != null) {
 +      ZooLock.deleteLock(tserverPath + "/" + args[1]);
 +    } else {
 +      System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
 +    }
 +    
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
index 13bcdbe,0000000..f1fdde4
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
 +import java.io.DataOutput;
 +import java.io.IOException;
 +
 +import org.apache.accumulo.core.data.Value;
 +
 +/**
 + * 
 + */
 +public class MemValue extends Value {
 +  int kvCount;
 +  boolean merged = false;
 +  
 +  /**
 +   * @param value
 +   *          Value
 +   * @param kv
 +   *          kv count
 +   */
 +  public MemValue(byte[] value, int kv) {
 +    super(value);
 +    this.kvCount = kv;
 +  }
 +  
 +  public MemValue() {
 +    super();
 +    this.kvCount = Integer.MAX_VALUE;
 +  }
 +  
 +  public MemValue(Value value, int kv) {
 +    super(value);
 +    this.kvCount = kv;
 +  }
 +  
 +  // Override
++  @Override
 +  public void write(final DataOutput out) throws IOException {
 +    if (!merged) {
 +      byte[] combinedBytes = new byte[getSize() + 4];
 +      System.arraycopy(value, 0, combinedBytes, 4, getSize());
 +      combinedBytes[0] = (byte) (kvCount >>> 24);
 +      combinedBytes[1] = (byte) (kvCount >>> 16);
 +      combinedBytes[2] = (byte) (kvCount >>> 8);
 +      combinedBytes[3] = (byte) (kvCount);
 +      value = combinedBytes;
 +      merged = true;
 +    }
 +    super.write(out);
 +  }
 +  
++  @Override
 +  public void set(final byte[] b) {
 +    super.set(b);
 +    merged = false;
 +  }
 +
++  @Override
 +  public void copy(byte[] b) {
 +    super.copy(b);
 +    merged = false;
 +  }
 +  
 +  /**
 +   * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
 +   * 
-    * @param v
 +   * @return The kvCount embedded in v.
 +   */
 +  public static int splitKVCount(Value v) {
 +    if (v instanceof MemValue)
 +      return ((MemValue) v).kvCount;
 +    
 +    byte[] originalBytes = new byte[v.getSize() - 4];
 +    byte[] combined = v.get();
 +    System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
 +    v.set(originalBytes);
 +    return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
 +  }
 +}


Mime
View raw message