Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E13BEEC2F for ; Thu, 28 Feb 2013 00:38:41 +0000 (UTC) Received: (qmail 84076 invoked by uid 500); 28 Feb 2013 00:38:41 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 84028 invoked by uid 500); 28 Feb 2013 00:38:41 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 84021 invoked by uid 99); 28 Feb 2013 00:38:41 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Feb 2013 00:38:41 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Feb 2013 00:38:35 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8749B23889B3; Thu, 28 Feb 2013 00:38:16 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1451062 [2/2] - in /accumulo/branches/ACCUMULO-259-polishing: ./ core/ core/src/main/java/org/apache/accumulo/core/cli/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/impl/thrif... Date: Thu, 28 Feb 2013 00:38:14 -0000 To: commits@accumulo.apache.org From: ctubbsii@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130228003816.8749B23889B3@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Feb 28 00:38:13 2013 @@ -81,6 +81,7 @@ import org.apache.accumulo.core.security import org.apache.accumulo.core.security.thrift.SecurityErrorCode; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.security.thrift.ThriftSecurityException; +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.Daemon; @@ -101,7 +102,7 @@ import org.apache.accumulo.server.conf.S import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection; import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer; import org.apache.accumulo.server.master.balancer.TabletBalancer; -import org.apache.accumulo.server.master.recovery.RecoverLease; +import org.apache.accumulo.server.master.recovery.RecoveryManager; import org.apache.accumulo.server.master.state.Assignment; import org.apache.accumulo.server.master.state.CurrentState; import org.apache.accumulo.server.master.state.DeadServerList; @@ -157,7 +158,6 @@ import org.apache.accumulo.start.classlo import org.apache.accumulo.trace.instrument.thrift.TraceWrap; import org.apache.accumulo.trace.thrift.TInfo; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; @@ -202,6 +202,7 @@ public class Master implements LiveTServ final private SortedMap migrations = Collections.synchronizedSortedMap(new TreeMap()); final private EventCoordinator nextEvent = new EventCoordinator(); final private Object mergeLock = new Object(); + private RecoveryManager recoveryManager = null; private ZooLock masterLock = null; private TServer clientService = null; @@ -227,15 +228,14 @@ public class Master implements LiveTServ static final boolean X = true; static final boolean _ = false; static final boolean transitionOK[][] = { - // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP - /* INITIAL */ {X, X, _, _, _, _, X}, - /* HAVE_LOCK */ {_, X, X, X, _, _, X}, - /* SAFE_MODE */ {_, _, X, X, X, _, X}, - /* NORMAL */ {_, _, X, X, X, _, X}, - /* UNLOAD_METADATA_TABLETS */ {_, _, X, X, X, X, X}, - /* UNLOAD_ROOT_TABLET */ {_, _, _, _, _, X, X}, - /* STOP */ {_, _, _, _, _, _, X} - }; + // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP + /* INITIAL */{X, X, _, _, _, _, X}, + /* HAVE_LOCK */{_, X, X, X, _, _, X}, + /* SAFE_MODE */{_, _, X, X, X, _, X}, + /* NORMAL */{_, _, X, X, X, _, X}, + /* UNLOAD_METADATA_TABLETS */{_, _, X, X, X, X, X}, + /* UNLOAD_ROOT_TABLET */{_, _, _, X, _, X, X}, + /* STOP */{_, _, _, _, _, _, X}}; synchronized private void setMasterState(MasterState newState) { if (state.equals(newState)) @@ -275,11 +275,11 @@ public class Master implements LiveTServ log.info("Upgrading zookeeper"); IZooReaderWriter zoo = ZooReaderWriter.getInstance(); - + zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP); - + for (String id : Tables.getIdToNameMap(instance).keySet()) { - + zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(), NodeExistsPolicy.SKIP); } @@ -289,7 +289,7 @@ public class Master implements LiveTServ } } } - + private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false); private final ServerConfiguration serverConfig; @@ -710,7 +710,8 @@ public class Master implements LiveTServ @Override public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName, TabletSplit split) throws TException { - if (migrations.remove(new KeyExtent(split.oldTablet)) != null) { + KeyExtent oldTablet = new KeyExtent(split.oldTablet); + if (migrations.remove(oldTablet) != null) { log.info("Canceled migration of " + split.oldTablet); } for (TServerInstance instance : tserverSet.getCurrentServers()) { @@ -1341,7 +1342,7 @@ public class Master implements LiveTServ if (goal == TabletGoalState.HOSTED) { if (state != TabletState.HOSTED && !tls.walogs.isEmpty()) { - if (recoverLogs(tls.extent, tls.walogs)) + if (recoveryManager.recoverLogs(tls.extent, tls.walogs)) continue; } switch (state) { @@ -1469,6 +1470,8 @@ public class Master implements LiveTServ } else { log.warn("Not connected to server " + tls.current); } + } catch (NotServingTabletException e) { + log.debug("Error asking tablet server to split a tablet: " + e); } catch (Exception e) { log.warn("Error asking tablet server to split a tablet: " + e); } @@ -1858,13 +1861,7 @@ public class Master implements LiveTServ try { switch (getMasterGoalState()) { case NORMAL: - switch (getMasterState()) { - case HAVE_LOCK: - case SAFE_MODE: - setMasterState(MasterState.NORMAL); - default: - break; - } + setMasterState(MasterState.NORMAL); break; case SAFE_MODE: if (getMasterState() == MasterState.NORMAL) { @@ -2002,7 +1999,10 @@ public class Master implements LiveTServ String oldName = t.getName(); try { t.setName("Getting status from " + server); - TabletServerStatus status = tserverSet.getConnection(server).getTableMap(); + TServerConnection connection = tserverSet.getConnection(server); + if (connection == null) + throw new IOException("No connection to " + server); + TabletServerStatus status = connection.getTableMap(); // TODO maybe remove from bad servers result.put(server, status); } finally { @@ -2034,39 +2034,13 @@ public class Master implements LiveTServ return result; } - public boolean recoverLogs(KeyExtent extent, Collection> walogs) throws IOException { - boolean recoveryNeeded = false; - for (Collection logs : walogs) { - for (String log : logs) { - String parts[] = log.split("/"); - String host = parts[0]; - String filename = parts[1]; - if (fs.exists(new Path(Constants.getRecoveryDir(getSystemConfiguration()) + "/" + filename + "/finished"))) { - recoveriesInProgress.remove(filename); - continue; - } - recoveryNeeded = true; - synchronized (recoveriesInProgress) { - if (!recoveriesInProgress.contains(filename)) { - Master.log.info("Starting recovery of " + filename + " created for " + host + ", tablet " + extent + " holds a reference"); - AccumuloConfiguration aconf = getConfiguration().getConfiguration(); - RecoverLease impl = createInstanceFromPropertyName(aconf, Property.MASTER_LEASE_RECOVERY_IMPLEMETATION, RecoverLease.class, new RecoverLease()); - impl.init(host, filename); - long tid = fate.startTransaction(); - fate.seedTransaction(tid, impl, true); - recoveriesInProgress.add(filename); - } - } - } - } - return recoveryNeeded; - } - public void run() throws IOException, InterruptedException, KeeperException { final String zroot = ZooUtil.getRoot(instance); getMasterLock(zroot + Constants.ZMASTER_LOCK); + recoveryManager = new RecoveryManager(this); + TableManager.getInstance().addObserver(this); StatusThread statusThread = new StatusThread(); Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java Thu Feb 28 00:38:13 2013 @@ -103,7 +103,8 @@ import org.apache.thrift.TException; */ public class BulkImport extends MasterRepo { - + public static final String FAILURES_TXT = "failures.txt"; + private static final long serialVersionUID = 1L; private static final Logger log = Logger.getLogger(BulkImport.class); @@ -363,13 +364,13 @@ class CopyFailed extends MasterRepo { FileSystem fs = master.getFileSystem(); - if (!fs.exists(new Path(error, "failures.txt"))) + if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT))) return new CleanUpBulkImport(tableId, source, bulk, error); HashMap failures = new HashMap(); HashMap loadedFailures = new HashMap(); - FSDataInputStream failFile = fs.open(new Path(error, "failures.txt")); + FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT)); BufferedReader in = new BufferedReader(new InputStreamReader(failFile)); try { String line = null; @@ -408,7 +409,7 @@ class CopyFailed extends MasterRepo { Path orig = new Path(failure); Path dest = new Path(error, orig.getName()); fs.rename(orig, dest); - log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed"); + log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed"); } if (loadedFailures.size() > 0) { @@ -432,7 +433,7 @@ class CopyFailed extends MasterRepo { bifCopyQueue.waitUntilDone(workIds); } - fs.delete(new Path(error, "failures.txt"), true); + fs.delete(new Path(error, BulkImport.FAILURES_TXT), true); return new CleanUpBulkImport(tableId, source, bulk, error); } @@ -539,7 +540,7 @@ class LoadFiles extends MasterRepo { failures.addAll(fail); } } catch (Exception ex) { - log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex, ex); + log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex); } finally { ServerClient.close(client); } @@ -556,7 +557,7 @@ class LoadFiles extends MasterRepo { } } - FSDataOutputStream failFile = fs.create(new Path(errorDir, "failures.txt"), true); + FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true); BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile)); try { for (String f : filesToLoad) { Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java Thu Feb 28 00:38:13 2013 @@ -92,7 +92,7 @@ class TableRangeOpWait extends MasterRep public Repo call(long tid, Master master) throws Exception { Text tableIdText = new Text(tableId); MergeInfo mergeInfo = master.getMergeInfo(tableIdText); - log.warn("removing merge information " + mergeInfo); + log.info("removing merge information " + mergeInfo); master.clearMergeState(tableIdText); Utils.unreserveTable(tableId, tid, true); // We can't add entries to the metadata table if it is offline for this merge. @@ -159,7 +159,7 @@ public class TableRangeOp extends Master Text tableIdText = new Text(tableId); MergeInfo mergeInfo = env.getMergeInfo(tableIdText); if (mergeInfo.getState() != MergeState.NONE) - log.warn("removing merge information " + mergeInfo); + log.info("removing merge information " + mergeInfo); env.clearMergeState(tableIdText); Utils.unreserveTable(tableId, tid, true); } Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Thu Feb 28 00:38:13 2013 @@ -2353,9 +2353,8 @@ public class TabletServer extends Abstra log.debug("Failed to unload tablet " + extent + "... it was alread closing or closed : " + e.getMessage()); } else { log.error("Failed to close tablet " + extent + "... Aborting migration", e); + enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent)); } - - enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent)); return; } Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java Thu Feb 28 00:38:13 2013 @@ -103,7 +103,7 @@ public class LogSorter { String formerThreadName = Thread.currentThread().getName(); int part = 0; try { - + // the following call does not throw an exception if the file/dir does not exist fs.delete(new Path(destPath), true); @@ -182,7 +182,7 @@ public class LogSorter { Thread.currentThread().setName(formerThreadName); try { close(); - } catch (IOException e) { + } catch (Exception e) { log.error("Error during cleanup sort/copy " + name, e); } synchronized (this) { Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java Thu Feb 28 00:38:13 2013 @@ -16,6 +16,7 @@ */ package org.apache.accumulo.server.zookeeper; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; @@ -106,16 +107,16 @@ public class DistributedWorkQueue { log.error("Error received when trying to delete entry in zookeeper " + childPath, e); } - try { - zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP); - } catch (Exception e) { - log.error("Error received when trying to delete entry in zookeeper " + childPath, e); - } - } catch (Exception e) { log.warn("Failed to process work " + child, e); } + try { + zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP); + } catch (Exception e) { + log.error("Error received when trying to delete entry in zookeeper " + childPath, e); + } + } finally { numTask.decrementAndGet(); } @@ -210,6 +211,12 @@ public class DistributedWorkQueue { zoo.putPersistentData(path + "/" + workId, data, NodeExistsPolicy.SKIP); } + public List getWorkQueued() throws KeeperException, InterruptedException { + ArrayList children = new ArrayList(zoo.getChildren(path)); + children.remove(LOCKS_NODE); + return children; + } + public void waitUntilDone(Set workIDs) throws KeeperException, InterruptedException { final String condVar = new String("cond"); Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java Thu Feb 28 00:38:13 2013 @@ -204,7 +204,6 @@ public class TestIngest { BatchWriter bw = null; FileSKVWriter writer = null; - Connector connector = opts.getConnector(); if (opts.outputFile != null) { Configuration conf = CachedConfiguration.getInstance(); @@ -213,9 +212,10 @@ public class TestIngest { AccumuloConfiguration.getDefaultConfiguration()); writer.startDefaultLocalityGroup(); } else { + Connector connector = opts.getConnector(); bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig()); + connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS); } - connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS); Text labBA = new Text(opts.columnVisibility.getExpression()); long startTime = System.currentTimeMillis(); Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java Thu Feb 28 00:38:13 2013 @@ -21,9 +21,7 @@ import java.util.Iterator; import java.util.Map.Entry; import java.util.Random; -import org.apache.accumulo.trace.instrument.Trace; import org.apache.accumulo.core.cli.ScannerOpts; -import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.Scanner; @@ -33,14 +31,13 @@ import org.apache.accumulo.core.data.Ran import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.trace.DistributedTrace; -import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.fate.zookeeper.ZooReader; +import org.apache.accumulo.trace.instrument.Trace; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; import com.beust.jcommander.Parameter; - public class VerifyIngest { private static final Logger log = Logger.getLogger(VerifyIngest.class); @@ -54,7 +51,7 @@ public class VerifyIngest { } public static class Opts extends TestIngest.Opts { - @Parameter(names="-useGet", description="fetches values one at a time, instead of scanning") + @Parameter(names = "-useGet", description = "fetches values one at a time, instead of scanning") boolean useGet = false; } @@ -71,19 +68,12 @@ public class VerifyIngest { Trace.currentTrace().data("cmdLine", Arrays.asList(args).toString()); } - Connector connector = null; - while (connector == null) { - try { - connector = opts.getConnector(); - } catch (AccumuloException e) { - log.warn("Could not connect to accumulo; will retry: " + e); - UtilWaitThread.sleep(1000); - } - } + Connector connector = opts.getConnector(); byte[][] bytevals = TestIngest.generateValues(opts); Authorizations labelAuths = new Authorizations("L1", "L2", "G1", "GROUP2"); + connector.securityOperations().changeUserAuthorizations(opts.principal, labelAuths); int expectedRow = opts.startRow; int expectedCol = 0; Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java Thu Feb 28 00:38:13 2013 @@ -21,6 +21,8 @@ import java.util.List; import java.util.Map; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.security.tokens.PasswordToken; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.test.CreateRFiles; @@ -62,8 +64,8 @@ public class BulkSplitOptimizationTest e FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); fs.delete(new Path("/tmp/testmf"), true); - - CreateRFiles.main(new String[] { "--output", "tmp/testmf", "--numThreads", "8", "--start", "0", "--end", "100000", "--splits", "99"}); + AuthenticationToken token = this.getToken(); + CreateRFiles.main(new String[] {"--output", "tmp/testmf", "--numThreads", "8", "--start", "0", "--end", "100000", "--splits", "99"}); bulkImport(fs, TABLE_NAME, "/tmp/testmf"); @@ -82,7 +84,11 @@ public class BulkSplitOptimizationTest e checkSplits(TABLE_NAME, 50, 100); - VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56", "--rows", "100000", "--start", "0", "--cols", "1"}); + String passwd = ""; + if (token instanceof PasswordToken) { + passwd = new String(((PasswordToken) token).getPassword()); + } + VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56", "--rows", "100000", "--start", "0", "--cols", "1", "-p", passwd}); // ensure each tablet does not have all map files checkRFiles(TABLE_NAME, 50, 100, 1, 4); Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java Thu Feb 28 00:38:13 2013 @@ -110,8 +110,10 @@ public class Config extends Test { String table = parts[0]; int choice = Integer.parseInt(parts[1]); Property property = tableSettings[choice].property; - log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue()); - state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue()); + if (state.getConnector().tableOperations().exists(table)) { + log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue()); + state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue()); + } } state.getMap().remove(LAST_SETTING); state.getMap().remove(LAST_TABLE_SETTING); @@ -136,9 +138,9 @@ public class Config extends Test { // generate a random value long newValue = random.nextLong(setting.min, setting.max); - state.getMap().put(LAST_TABLE_SETTING, "" + choice); + state.getMap().put(LAST_TABLE_SETTING, table + "," + choice); log.debug("Setting " + setting.property.getKey() + " on table " + table + " to " + newValue); - state.getConnector().instanceOperations().setProperty(setting.property.getKey(), table + "," + newValue); + state.getConnector().tableOperations().setProperty(table, setting.property.getKey(), "" + newValue); } private void changeSetting(RandomData random, State state, Properties props) throws Exception { Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java Thu Feb 28 00:38:13 2013 @@ -21,6 +21,7 @@ import java.util.Set; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.security.tokens.PasswordToken; @@ -52,7 +53,7 @@ public class SecurityFixture extends Fix conn.securityOperations().createLocalUser(systemUserName, sysUserPass); WalkingSecurity.get(state).setTableName(secTableName); - state.set("rootUserPass", state.getCredentials().getToken()); + state.set("rootUserPass", CredentialHelper.extractToken(state.getCredentials())); WalkingSecurity.get(state).setSysUserName(systemUserName); WalkingSecurity.get(state).createUser(systemUserName, sysUserPass); Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java Thu Feb 28 00:38:13 2013 @@ -279,7 +279,7 @@ public class WalkingSecurity extends Sec } public byte[] getUserPassword(String user) { - Object obj = state.get(getSysUserName() + userPass); + Object obj = state.get(user + userPass); if (obj instanceof PasswordToken) { return ((PasswordToken) obj).getPassword(); } @@ -295,7 +295,11 @@ public class WalkingSecurity extends Sec } public byte[] getTabPassword() { - return (byte[]) state.get(getTabUserName() + userPass); + Object obj = state.get(getTabUserName() + userPass); + if (obj instanceof PasswordToken) { + return ((PasswordToken) obj).getPassword(); + } + return null; } public boolean userPassTransient(String user) { Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py Thu Feb 28 00:38:13 2013 @@ -120,7 +120,7 @@ class TestUtilsMixin: ld = 'ACCUMULO_LOG_DIR=%s/logs/%s' % (ACCUMULO_HOME, ID) execcmd = ['ssh', '-q', host, cp, jo, go, ld] + quote(cmd) log.debug(repr(execcmd)) - return Popen(execcmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, **opts) + return Popen(execcmd, stdout=PIPE, stderr=PIPE, **opts) def shell(self, host, input, **opts): """Run accumulo shell with the given input, @@ -220,7 +220,7 @@ class TestUtilsMixin: args = '' if timestamp: args += "-ts %ld " % int(timestamp) - args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1 --createTable ' % (INSTANCE_NAME, ROOT, size, count, start) + args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1 --createTable -p %s' % (INSTANCE_NAME, ROOT, size, count, start, ROOT_PASSWORD) if colf: args = '--columnFamily %s ' % colf + args return self.runClassOn(host, klass, args.split(), **kwargs) @@ -230,7 +230,7 @@ class TestUtilsMixin: args = '' if timestamp: args += "-ts %ld " % int(timestamp) - args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 ' % (INSTANCE_NAME, ROOT, size, colf, count, start) + args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 -p %s' % (INSTANCE_NAME, ROOT, size, colf, count, start, ROOT_PASSWORD) return self.runClassOn(host, klass, args.split()) def stop_accumulo(self, signal=signal.SIGHUP): Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py Thu Feb 28 00:38:13 2013 @@ -36,7 +36,7 @@ class BinaryTest(unittest.TestCase, Test handle = self.runClassOn(self.masterHost(), testClass, ['-u', 'root', '--mode', mode, '-t', table, - '--start', min, '--count', max]) + '--start', min, '--count', max, '-p', 'secret']) self.waitForStop(handle, 200) def tearDown(self): Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py Thu Feb 28 00:38:13 2013 @@ -63,7 +63,7 @@ class Examples(TestUtilsMixin, unittest. self.comment("Testing MaxMutation constraint") self.ashell('createtable test_ingest\n' 'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n') - handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000']) + handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD]) out, err = handle.communicate() self.failIf(handle.returncode==0) self.failUnless(err.find("MutationsRejectedException: # constraint violations : 1") >= 0, "Was able to insert a mutation larger than max size") Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py Thu Feb 28 00:38:13 2013 @@ -20,7 +20,7 @@ import unittest import time import sys -from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME +from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME, ROOT, ROOT_PASSWORD log = logging.getLogger('test.auto') @@ -82,12 +82,12 @@ class MultiTableTest(SunnyDayTest): def ingest(self, host, count, *args, **kwargs): klass = 'org.apache.accumulo.test.TestMultiTableIngest' - args = '--count %d -u root -i %s ' % (count, INSTANCE_NAME) + args = '--count %d -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD) return self.runClassOn(host, klass, args.split()) def verify(self, host, count, *args, **kwargs): klass = 'org.apache.accumulo.test.TestMultiTableIngest' - args = '--count %d --readonly -u root -i %s ' % (count, INSTANCE_NAME) + args = '--count %d --readonly -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD) return self.runClassOn(host, klass, args.split()) Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py Thu Feb 28 00:38:13 2013 @@ -19,7 +19,7 @@ import logging import unittest import time -from TestUtils import TestUtilsMixin +from TestUtils import TestUtilsMixin, ROOT_PASSWORD log = logging.getLogger('test.auto') @@ -33,7 +33,7 @@ class WriteLots(unittest.TestCase, TestU def ingest(self, host, start, count, **kwargs): klass = 'org.apache.accumulo.test.TestIngest' - args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1' % (count, start) + args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1 -p %s' % (count, start, ROOT_PASSWORD) return self.runClassOn(host, klass, args.split(), **kwargs) def setUp(self): Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl?rev=1451062&r1=1451061&r2=1451062&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl (original) +++ accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl Thu Feb 28 00:38:13 2013 @@ -19,15 +19,38 @@ use POSIX qw(strftime); if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){ - print "Usage : agitator.pl [ ]\n"; + print "Usage : agitator.pl [:max sleep before kill in minutes] [:max sleep before tup in minutes] [ ]\n"; exit(1); } $ACCUMULO_HOME="../../.."; $HADOOP_PREFIX=$ENV{"HADOOP_PREFIX"}; -$sleep1 = $ARGV[0]; -$sleep2 = $ARGV[1]; +@sleeprange1 = split(/:/, $ARGV[0]); +$sleep1 = $sleeprange1[0]; + +@sleeprange2 = split(/:/, $ARGV[1]); +$sleep2 = $sleeprange2[0]; + +if(scalar(@sleeprange1) > 1){ + $sleep1max = $sleeprange1[1] + 1; +}else{ + $sleep1max = $sleep1; +} + +if($sleep1 > $sleep1max){ + die("sleep1 > sleep1max $sleep1 > $sleep1max"); +} + +if(scalar(@sleeprange2) > 1){ + $sleep2max = $sleeprange2[1] + 1; +}else{ + $sleep2max = $sleep2; +} + +if($sleep2 > $sleep2max){ + die("sleep2 > sleep2max $sleep2 > $sleep2max"); +} if(scalar(@ARGV) == 4){ $minKill = $ARGV[2]; @@ -104,13 +127,15 @@ while(1){ } } - sleep($sleep2 * 60); + $nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2; + sleep($nextsleep2 * 60); $t = strftime "%Y%m%d %H:%M:%S", localtime; print STDERR "$t Running tup\n"; system("$ACCUMULO_HOME/bin/tup.sh"); print STDERR "$t Running start-dfs\n"; system("$HADOOP_PREFIX/bin/start-dfs.sh"); - sleep($sleep1 * 60); + $nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1; + sleep($nextsleep1 * 60); }