Return-Path: X-Original-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6FE749FCD for ; Thu, 13 Oct 2011 13:03:09 +0000 (UTC) Received: (qmail 19399 invoked by uid 500); 13 Oct 2011 13:03:09 -0000 Delivered-To: apmail-incubator-accumulo-commits-archive@incubator.apache.org Received: (qmail 19367 invoked by uid 500); 13 Oct 2011 13:03:09 -0000 Mailing-List: contact accumulo-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: accumulo-dev@incubator.apache.org Delivered-To: mailing list accumulo-commits@incubator.apache.org Received: (qmail 19360 invoked by uid 99); 13 Oct 2011 13:03:08 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Oct 2011 13:03:08 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Oct 2011 13:03:06 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B071F238889B; Thu, 13 Oct 2011 13:02:46 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1182826 - in /incubator/accumulo/trunk: bin/ src/core/src/main/java/org/apache/accumulo/core/util/ src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ src/server/src/main/java/org/apache/accumulo/server/master/ src/server/... Date: Thu, 13 Oct 2011 13:02:46 -0000 To: accumulo-commits@incubator.apache.org From: ecn@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111013130246.B071F238889B@eris.apache.org> Author: ecn Date: Thu Oct 13 13:02:45 2011 New Revision: 1182826 URL: http://svn.apache.org/viewvc?rev=1182826&view=rev Log: ACCUMULO-27: add the fixes from scale random-walk testing Modified: incubator/accumulo/trunk/bin/stop-server.sh incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/Duration.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveScanIterator.java incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java Modified: incubator/accumulo/trunk/bin/stop-server.sh URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/bin/stop-server.sh?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/bin/stop-server.sh (original) +++ incubator/accumulo/trunk/bin/stop-server.sh Thu Oct 13 13:02:45 2011 @@ -1,10 +1,16 @@ #! /usr/bin/env bash +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + + # only stop if there's not one already running if [ "$1" = "`hostname`" ]; then - PID=`ps -ef | egrep ${2} | grep "Main ${3}" | grep -v grep | grep -v ssh | grep -v stop-server.sh | awk {'print \$2'} | head -1` + PID=`ps -ef | grep "$ACCUMULO_HOME" | egrep ${2} | grep "Main ${3}" | grep -v grep | grep -v ssh | grep -v stop-server.sh | awk {'print \$2'} | head -1` else - PID=`ssh -q -o 'ConnectTimeout 8' $1 "ps -ef | egrep '${2}' | grep 'Main ${3}' | grep -v grep | grep -v ssh | grep -v stop-server.sh" | awk {'print $2'} | head -1` + PID=`ssh -q -o 'ConnectTimeout 8' $1 "ps -ef | grep \"$ACCUMULO_HOME\" egrep '${2}' | grep 'Main ${3}' | grep -v grep | grep -v ssh | grep -v stop-server.sh" | awk {'print $2'} | head -1` fi if [ ! -z $PID ]; then echo "stopping ${3} on $1"; Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/Duration.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/Duration.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/Duration.java (original) +++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/Duration.java Thu Oct 13 13:02:45 2011 @@ -7,10 +7,14 @@ public class Duration { } public static String format(long time, String space) { + return format(time, space, "&emdash;"); + } + + public static String format(long time, String space, String zero) { long ms, sec, min, hr, day, yr; ms = sec = min = hr = day = yr = -1; if (time == 0) - return "—"; + return zero; ms = time % 1000; time /= 1000; if (time == 0) @@ -33,6 +37,7 @@ public class Duration { return String.format("%dd"+space+"%dh", day, hr); yr = time; return String.format("%dy"+space+"%dd", yr, day); + } } Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveScanIterator.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveScanIterator.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveScanIterator.java (original) +++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveScanIterator.java Thu Oct 13 13:02:45 2011 @@ -28,7 +28,7 @@ class ActiveScanIterator implements Iter for (ActiveScan as : asl) scans.add(String.format("%21s |%21s |%9s |%9s |%7s |%6s |%8s |%8s |%10s |%10s |%10s | %s" - , tserver, as.getClient(), Duration.format(as.getAge(), ""), Duration.format(as.getLastContactTime(), ""), as.getState(), as.getType(), as.getUser(), as.getTable(), as.getColumns(), (as.getType() == ScanType.SINGLE ? as.getExtent() : "N/A"), as.getSsiList(), as.getSsio())); + , tserver, as.getClient(), Duration.format(as.getAge(), "", "-"), Duration.format(as.getLastContactTime(), "", "-"), as.getState(), as.getType(), as.getUser(), as.getTable(), as.getColumns(), (as.getType() == ScanType.SINGLE ? as.getExtent() : "N/A"), as.getSsiList(), as.getSsio())); } catch (Exception e) { Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java Thu Oct 13 13:02:45 2011 @@ -221,7 +221,7 @@ public class LiveTServerSet implements W } } // log.debug("Current: " + current.keySet()); - if (doomed.size() + updates.size() > 0) + if (!doomed.isEmpty() && !updates.isEmpty()) this.cback.update(this, doomed, updates); } catch (Exception ex) { log.error(ex, ex); Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Oct 13 13:02:45 2011 @@ -1701,6 +1701,11 @@ public class Master implements LiveTServ } log.debug("Moved " + fileCount + " files to " + stop); + if (firstPrevRowValue == null) { + log.debug("tablet already merged"); + return; + } + stop.setPrevEndRow(KeyExtent.decodePrevEndRow(firstPrevRowValue)); Mutation updatePrevRow = stop.getPrevRowUpdateMutation(); log.debug("Setting the prevRow for last tablet: " + stop); @@ -2021,8 +2026,6 @@ public class Master implements LiveTServ long start = System.currentTimeMillis(); SortedMap result = new TreeMap(); for (TServerInstance server : tserverSet.getCurrentServers()) { - if (serversToShutdown.contains(server)) - continue; try { TabletServerStatus status = tserverSet.getConnection(server).getTableMap(); result.put(server, status); Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java Thu Oct 13 13:02:45 2011 @@ -115,7 +115,8 @@ public class TableRangeOp extends Master // Not sure this is a good thing to do. The Master state engine should be the one to remove it. Text tableIdText = new Text(tableId); MergeInfo mergeInfo = env.getMergeInfo(tableIdText); - log.warn("removing merge information " + mergeInfo); + if (mergeInfo.getState() != MergeState.NONE) + log.warn("removing merge information " + mergeInfo); env.clearMergeState(tableIdText); Utils.unreserveTable(tableId, tid, true); } Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java Thu Oct 13 13:02:45 2011 @@ -48,15 +48,15 @@ public class DisconnectLogger extends Ma MutationLogger.Iface client = ThriftUtil.getClient(new MutationLogger.Client.Factory(), addr, ServerConfiguration.getSystemConfiguration()); try { client.beginShutdown(null, SecurityConstants.getSystemCredentials()); - String zpath = ZooUtil.getRoot(m.getInstance()) + Constants.ZLOGGERS + "/" + entry.getKey(); - ZooReaderWriter.getInstance().recursiveDelete(zpath, NodeMissingPolicy.SKIP); - log.info("logger asked to halt " + location); - return new FlushTablets(entry.getValue()); } catch (Exception ex) { log.error("Unable to talk to logger at " + addr); } finally { ThriftUtil.returnClient(client); } + String zpath = ZooUtil.getRoot(m.getInstance()) + Constants.ZLOGGERS + "/" + entry.getKey(); + ZooReaderWriter.getInstance().recursiveDelete(zpath, NodeMissingPolicy.SKIP); + log.info("logger asked to halt " + location); + return new FlushTablets(entry.getValue()); } } } while (m.stillMaster() && foundMatch); Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java?rev=1182826&r1=1182825&r2=1182826&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java Thu Oct 13 13:02:45 2011 @@ -374,9 +374,10 @@ public final class ZKAuthenticator imple if (user.equals(getRootUsername()) || user.equals(SecurityConstants.SYSTEM_USERNAME)) return true; - if (userExists(user)) + byte[] perms = zooCache.get(ZKUserPath+"/"+user+ZKUserSysPerms); + if (perms != null) { - if (Tool.convertSystemPermissions(zooCache.get(ZKUserPath+"/"+user+ZKUserSysPerms)).contains(permission)) + if (Tool.convertSystemPermissions(perms).contains(permission)) return true; zooCache.clear(ZKUserPath+"/"+user+ZKUserSysPerms); return Tool.convertSystemPermissions(zooCache.get(ZKUserPath+"/"+user+ZKUserSysPerms)).contains(permission);