accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject svn commit: r1451062 [2/2] - in /accumulo/branches/ACCUMULO-259-polishing: ./ core/ core/src/main/java/org/apache/accumulo/core/cli/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/impl/thrif...
Date Thu, 28 Feb 2013 00:38:14 GMT
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java
Thu Feb 28 00:38:13 2013
@@ -81,6 +81,7 @@ import org.apache.accumulo.core.security
 import org.apache.accumulo.core.security.thrift.SecurityErrorCode;
 import org.apache.accumulo.core.security.thrift.TCredentials;
 import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 import org.apache.accumulo.core.util.ByteBufferUtil;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.Daemon;
@@ -101,7 +102,7 @@ import org.apache.accumulo.server.conf.S
 import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
 import org.apache.accumulo.server.master.balancer.TabletBalancer;
-import org.apache.accumulo.server.master.recovery.RecoverLease;
+import org.apache.accumulo.server.master.recovery.RecoveryManager;
 import org.apache.accumulo.server.master.state.Assignment;
 import org.apache.accumulo.server.master.state.CurrentState;
 import org.apache.accumulo.server.master.state.DeadServerList;
@@ -157,7 +158,6 @@ import org.apache.accumulo.start.classlo
 import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
 import org.apache.accumulo.trace.thrift.TInfo;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
@@ -202,6 +202,7 @@ public class Master implements LiveTServ
   final private SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new
TreeMap<KeyExtent,TServerInstance>());
   final private EventCoordinator nextEvent = new EventCoordinator();
   final private Object mergeLock = new Object();
+  private RecoveryManager recoveryManager = null;
   
   private ZooLock masterLock = null;
   private TServer clientService = null;
@@ -227,15 +228,14 @@ public class Master implements LiveTServ
   static final boolean X = true;
   static final boolean _ = false;
   static final boolean transitionOK[][] = {
-      //                            INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT
STOP
-      /* INITIAL */                 {X,     X,        _,        _,     _,          _,   
      X},
-      /* HAVE_LOCK */               {_,     X,        X,        X,     _,          _,   
      X},
-      /* SAFE_MODE */               {_,     _,        X,        X,     X,          _,   
      X},
-      /* NORMAL */                  {_,     _,        X,        X,     X,          _,   
      X},
-      /* UNLOAD_METADATA_TABLETS */ {_,     _,        X,        X,     X,          X,   
      X},
-      /* UNLOAD_ROOT_TABLET */      {_,     _,        _,        _,     _,          X,   
      X},
-      /* STOP */                    {_,     _,        _,        _,     _,          _,   
      X}
-  };
+      // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
+      /* INITIAL */{X, X, _, _, _, _, X},
+      /* HAVE_LOCK */{_, X, X, X, _, _, X},
+      /* SAFE_MODE */{_, _, X, X, X, _, X},
+      /* NORMAL */{_, _, X, X, X, _, X},
+      /* UNLOAD_METADATA_TABLETS */{_, _, X, X, X, X, X},
+      /* UNLOAD_ROOT_TABLET */{_, _, _, X, _, X, X},
+      /* STOP */{_, _, _, _, _, _, X}};
   
   synchronized private void setMasterState(MasterState newState) {
     if (state.equals(newState))
@@ -275,11 +275,11 @@ public class Master implements LiveTServ
         log.info("Upgrading zookeeper");
         
         IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-
+        
         zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
-
+        
         for (String id : Tables.getIdToNameMap(instance).keySet()) {
-
+          
           zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(),
               NodeExistsPolicy.SKIP);
         }
@@ -289,7 +289,7 @@ public class Master implements LiveTServ
       }
     }
   }
-
+  
   private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
   
   private final ServerConfiguration serverConfig;
@@ -710,7 +710,8 @@ public class Master implements LiveTServ
     
     @Override
     public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName,
TabletSplit split) throws TException {
-      if (migrations.remove(new KeyExtent(split.oldTablet)) != null) {
+      KeyExtent oldTablet = new KeyExtent(split.oldTablet);
+      if (migrations.remove(oldTablet) != null) {
         log.info("Canceled migration of " + split.oldTablet);
       }
       for (TServerInstance instance : tserverSet.getCurrentServers()) {
@@ -1341,7 +1342,7 @@ public class Master implements LiveTServ
             
             if (goal == TabletGoalState.HOSTED) {
               if (state != TabletState.HOSTED && !tls.walogs.isEmpty()) {
-                if (recoverLogs(tls.extent, tls.walogs))
+                if (recoveryManager.recoverLogs(tls.extent, tls.walogs))
                   continue;
               }
               switch (state) {
@@ -1469,6 +1470,8 @@ public class Master implements LiveTServ
             } else {
               log.warn("Not connected to server " + tls.current);
             }
+          } catch (NotServingTabletException e) {
+            log.debug("Error asking tablet server to split a tablet: " + e);
           } catch (Exception e) {
             log.warn("Error asking tablet server to split a tablet: " + e);
           }
@@ -1858,13 +1861,7 @@ public class Master implements LiveTServ
         try {
           switch (getMasterGoalState()) {
             case NORMAL:
-              switch (getMasterState()) {
-                case HAVE_LOCK:
-                case SAFE_MODE:
-                  setMasterState(MasterState.NORMAL);
-                default:
-                  break;
-              }
+              setMasterState(MasterState.NORMAL);
               break;
             case SAFE_MODE:
               if (getMasterState() == MasterState.NORMAL) {
@@ -2002,7 +1999,10 @@ public class Master implements LiveTServ
         String oldName = t.getName();
         try {
           t.setName("Getting status from " + server);
-          TabletServerStatus status = tserverSet.getConnection(server).getTableMap();
+          TServerConnection connection = tserverSet.getConnection(server);
+          if (connection == null)
+            throw new IOException("No connection to " + server);
+          TabletServerStatus status = connection.getTableMap();
           // TODO maybe remove from bad servers
           result.put(server, status);
         } finally {
@@ -2034,39 +2034,13 @@ public class Master implements LiveTServ
     return result;
   }
   
-  public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>>
walogs) throws IOException {
-    boolean recoveryNeeded = false;
-    for (Collection<String> logs : walogs) {
-      for (String log : logs) {
-        String parts[] = log.split("/");
-        String host = parts[0];
-        String filename = parts[1];
-        if (fs.exists(new Path(Constants.getRecoveryDir(getSystemConfiguration()) + "/" +
filename + "/finished"))) {
-          recoveriesInProgress.remove(filename);
-          continue;
-        }
-        recoveryNeeded = true;
-        synchronized (recoveriesInProgress) {
-          if (!recoveriesInProgress.contains(filename)) {
-            Master.log.info("Starting recovery of " + filename + " created for " + host +
", tablet " + extent + " holds a reference");
-            AccumuloConfiguration aconf = getConfiguration().getConfiguration();
-            RecoverLease impl = createInstanceFromPropertyName(aconf, Property.MASTER_LEASE_RECOVERY_IMPLEMETATION,
RecoverLease.class, new RecoverLease());
-            impl.init(host, filename);
-            long tid = fate.startTransaction();
-            fate.seedTransaction(tid, impl, true);
-            recoveriesInProgress.add(filename);
-          }
-        }
-      }
-    }
-    return recoveryNeeded;
-  }
-  
   public void run() throws IOException, InterruptedException, KeeperException {
     final String zroot = ZooUtil.getRoot(instance);
     
     getMasterLock(zroot + Constants.ZMASTER_LOCK);
     
+    recoveryManager = new RecoveryManager(this);
+    
     TableManager.getInstance().addObserver(this);
     
     StatusThread statusThread = new StatusThread();

Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
Thu Feb 28 00:38:13 2013
@@ -103,7 +103,8 @@ import org.apache.thrift.TException;
  */
 
 public class BulkImport extends MasterRepo {
-  
+  public static final String FAILURES_TXT = "failures.txt";
+
   private static final long serialVersionUID = 1L;
   
   private static final Logger log = Logger.getLogger(BulkImport.class);
@@ -363,13 +364,13 @@ class CopyFailed extends MasterRepo {
 	  
     FileSystem fs = master.getFileSystem();
 	  
-    if (!fs.exists(new Path(error, "failures.txt")))
+    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
       return new CleanUpBulkImport(tableId, source, bulk, error);
     
     HashMap<String,String> failures = new HashMap<String,String>();
     HashMap<String,String> loadedFailures = new HashMap<String,String>();
     
-    FSDataInputStream failFile = fs.open(new Path(error, "failures.txt"));
+    FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
     BufferedReader in = new BufferedReader(new InputStreamReader(failFile));
     try {
       String line = null;
@@ -408,7 +409,7 @@ class CopyFailed extends MasterRepo {
       Path orig = new Path(failure);
       Path dest = new Path(error, orig.getName());
       fs.rename(orig, dest);
-      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed");
+      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
     }
     
     if (loadedFailures.size() > 0) {
@@ -432,7 +433,7 @@ class CopyFailed extends MasterRepo {
       bifCopyQueue.waitUntilDone(workIds);
     }
 
-    fs.delete(new Path(error, "failures.txt"), true);
+    fs.delete(new Path(error, BulkImport.FAILURES_TXT), true);
     return new CleanUpBulkImport(tableId, source, bulk, error);
   }
   
@@ -539,7 +540,7 @@ class LoadFiles extends MasterRepo {
                 failures.addAll(fail);
               }
             } catch (Exception ex) {
-              log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex, ex);
+              log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
             } finally {
               ServerClient.close(client);
             }
@@ -556,7 +557,7 @@ class LoadFiles extends MasterRepo {
       }
     }
     
-    FSDataOutputStream failFile = fs.create(new Path(errorDir, "failures.txt"), true);
+    FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT),
true);
     BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile));
     try {
       for (String f : filesToLoad) {

Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java
Thu Feb 28 00:38:13 2013
@@ -92,7 +92,7 @@ class TableRangeOpWait extends MasterRep
   public Repo<Master> call(long tid, Master master) throws Exception {
     Text tableIdText = new Text(tableId);
     MergeInfo mergeInfo = master.getMergeInfo(tableIdText);
-    log.warn("removing merge information " + mergeInfo);
+    log.info("removing merge information " + mergeInfo);
     master.clearMergeState(tableIdText);
     Utils.unreserveTable(tableId, tid, true);
     // We can't add entries to the metadata table if it is offline for this merge.
@@ -159,7 +159,7 @@ public class TableRangeOp extends Master
     Text tableIdText = new Text(tableId);
     MergeInfo mergeInfo = env.getMergeInfo(tableIdText);
     if (mergeInfo.getState() != MergeState.NONE)
-      log.warn("removing merge information " + mergeInfo);
+      log.info("removing merge information " + mergeInfo);
     env.clearMergeState(tableIdText);
     Utils.unreserveTable(tableId, tid, true);
   }

Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Thu Feb 28 00:38:13 2013
@@ -2353,9 +2353,8 @@ public class TabletServer extends Abstra
           log.debug("Failed to unload tablet " + extent + "... it was alread closing or closed
: " + e.getMessage());
         } else {
           log.error("Failed to close tablet " + extent + "... Aborting migration", e);
+          enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent));
         }
-        
-        enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent));
         return;
       }
       

Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Thu Feb 28 00:38:13 2013
@@ -103,7 +103,7 @@ public class LogSorter {
       String formerThreadName = Thread.currentThread().getName();
       int part = 0;
       try {
-        
+
         // the following call does not throw an exception if the file/dir does not exist
         fs.delete(new Path(destPath), true);
         
@@ -182,7 +182,7 @@ public class LogSorter {
         Thread.currentThread().setName(formerThreadName);
         try {
           close();
-        } catch (IOException e) {
+        } catch (Exception e) {
           log.error("Error during cleanup sort/copy " + name, e);
         }
         synchronized (this) {

Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
Thu Feb 28 00:38:13 2013
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.server.zookeeper;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -106,16 +107,16 @@ public class DistributedWorkQueue {
                   log.error("Error received when trying to delete entry in zookeeper " +
childPath, e);
                 }
                 
-                try {
-                  zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP);
-                } catch (Exception e) {
-                  log.error("Error received when trying to delete entry in zookeeper " +
childPath, e);
-                }
-                
               } catch (Exception e) {
                 log.warn("Failed to process work " + child, e);
               }
               
+              try {
+                zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP);
+              } catch (Exception e) {
+                log.error("Error received when trying to delete entry in zookeeper " + childPath,
e);
+              }
+
             } finally {
               numTask.decrementAndGet();
             }
@@ -210,6 +211,12 @@ public class DistributedWorkQueue {
     zoo.putPersistentData(path + "/" + workId, data, NodeExistsPolicy.SKIP);
   }
   
+  public List<String> getWorkQueued() throws KeeperException, InterruptedException
{
+    ArrayList<String> children = new ArrayList<String>(zoo.getChildren(path));
+    children.remove(LOCKS_NODE);
+    return children;
+  }
+
   public void waitUntilDone(Set<String> workIDs) throws KeeperException, InterruptedException
{
     
     final String condVar = new String("cond");

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java
Thu Feb 28 00:38:13 2013
@@ -204,7 +204,6 @@ public class TestIngest {
       
       BatchWriter bw = null;
       FileSKVWriter writer = null;
-      Connector connector = opts.getConnector();
       
       if (opts.outputFile != null) {
         Configuration conf = CachedConfiguration.getInstance();
@@ -213,9 +212,10 @@ public class TestIngest {
             AccumuloConfiguration.getDefaultConfiguration());
         writer.startDefaultLocalityGroup();
       } else {
+        Connector connector = opts.getConnector();
         bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
+        connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
       }
-      connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
       Text labBA = new Text(opts.columnVisibility.getExpression());
       
       long startTime = System.currentTimeMillis();

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
Thu Feb 28 00:38:13 2013
@@ -21,9 +21,7 @@ import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Random;
 
-import org.apache.accumulo.trace.instrument.Trace;
 import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.Scanner;
@@ -33,14 +31,13 @@ import org.apache.accumulo.core.data.Ran
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.trace.DistributedTrace;
-import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.trace.instrument.Trace;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 
 import com.beust.jcommander.Parameter;
 
-
 public class VerifyIngest {
   
   private static final Logger log = Logger.getLogger(VerifyIngest.class);
@@ -54,7 +51,7 @@ public class VerifyIngest {
   }
   
   public static class Opts extends TestIngest.Opts {
-    @Parameter(names="-useGet", description="fetches values one at a time, instead of scanning")
+    @Parameter(names = "-useGet", description = "fetches values one at a time, instead of
scanning")
     boolean useGet = false;
   }
   
@@ -71,19 +68,12 @@ public class VerifyIngest {
         Trace.currentTrace().data("cmdLine", Arrays.asList(args).toString());
       }
       
-      Connector connector = null;
-      while (connector == null) {
-        try {
-          connector = opts.getConnector();
-        } catch (AccumuloException e) {
-          log.warn("Could not connect to accumulo; will retry: " + e);
-          UtilWaitThread.sleep(1000);
-        }
-      }
+      Connector connector = opts.getConnector();
       
       byte[][] bytevals = TestIngest.generateValues(opts);
       
       Authorizations labelAuths = new Authorizations("L1", "L2", "G1", "GROUP2");
+      connector.securityOperations().changeUserAuthorizations(opts.principal, labelAuths);
       
       int expectedRow = opts.startRow;
       int expectedCol = 0;

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java
Thu Feb 28 00:38:13 2013
@@ -21,6 +21,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.tokens.PasswordToken;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.test.CreateRFiles;
@@ -62,8 +64,8 @@ public class BulkSplitOptimizationTest e
     
     FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
     fs.delete(new Path("/tmp/testmf"), true);
-    
-    CreateRFiles.main(new String[] { "--output", "tmp/testmf", "--numThreads", "8", "--start",
"0", "--end", "100000", "--splits", "99"});
+    AuthenticationToken token = this.getToken();
+    CreateRFiles.main(new String[] {"--output", "tmp/testmf", "--numThreads", "8", "--start",
"0", "--end", "100000", "--splits", "99"});
     
     bulkImport(fs, TABLE_NAME, "/tmp/testmf");
     
@@ -82,7 +84,11 @@ public class BulkSplitOptimizationTest e
     
     checkSplits(TABLE_NAME, 50, 100);
     
-    VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56",
"--rows", "100000", "--start", "0", "--cols", "1"});
+    String passwd = "";
+    if (token instanceof PasswordToken) {
+      passwd = new String(((PasswordToken) token).getPassword());
+    }
+    VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56",
"--rows", "100000", "--start", "0", "--cols", "1", "-p", passwd});
     
     // ensure each tablet does not have all map files
     checkRFiles(TABLE_NAME, 50, 100, 1, 4);

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
Thu Feb 28 00:38:13 2013
@@ -110,8 +110,10 @@ public class Config extends Test {
       String table = parts[0];
       int choice = Integer.parseInt(parts[1]);
       Property property = tableSettings[choice].property;
-      log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue());
-      state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue());
+      if (state.getConnector().tableOperations().exists(table)) {
+        log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue());
+        state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue());
+      }
     }
     state.getMap().remove(LAST_SETTING);
     state.getMap().remove(LAST_TABLE_SETTING);
@@ -136,9 +138,9 @@ public class Config extends Test {
     
     // generate a random value
     long newValue = random.nextLong(setting.min, setting.max);
-    state.getMap().put(LAST_TABLE_SETTING, "" + choice);
+    state.getMap().put(LAST_TABLE_SETTING, table + "," + choice);
     log.debug("Setting " + setting.property.getKey() + " on table " + table + " to " + newValue);
-    state.getConnector().instanceOperations().setProperty(setting.property.getKey(), table
+ "," + newValue);
+    state.getConnector().tableOperations().setProperty(table, setting.property.getKey(),
"" + newValue);
   }
   
   private void changeSetting(RandomData random, State state, Properties props) throws Exception
{

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
Thu Feb 28 00:38:13 2013
@@ -21,6 +21,7 @@ import java.util.Set;
 
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
 import org.apache.accumulo.core.security.SystemPermission;
 import org.apache.accumulo.core.security.TablePermission;
 import org.apache.accumulo.core.security.tokens.PasswordToken;
@@ -52,7 +53,7 @@ public class SecurityFixture extends Fix
     conn.securityOperations().createLocalUser(systemUserName, sysUserPass);
     
     WalkingSecurity.get(state).setTableName(secTableName);
-    state.set("rootUserPass", state.getCredentials().getToken());
+    state.set("rootUserPass", CredentialHelper.extractToken(state.getCredentials()));
     
     WalkingSecurity.get(state).setSysUserName(systemUserName);
     WalkingSecurity.get(state).createUser(systemUserName, sysUserPass);

Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java
(original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java
Thu Feb 28 00:38:13 2013
@@ -279,7 +279,7 @@ public class WalkingSecurity extends Sec
   }
   
   public byte[] getUserPassword(String user) {
-    Object obj = state.get(getSysUserName() + userPass);
+    Object obj = state.get(user + userPass);
     if (obj instanceof PasswordToken) {
       return ((PasswordToken) obj).getPassword();
     }
@@ -295,7 +295,11 @@ public class WalkingSecurity extends Sec
   }
   
   public byte[] getTabPassword() {
-    return (byte[]) state.get(getTabUserName() + userPass);
+    Object obj = state.get(getTabUserName() + userPass);
+    if (obj instanceof PasswordToken) {
+      return ((PasswordToken) obj).getPassword();
+    }
+    return null;
   }
   
   public boolean userPassTransient(String user) {

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py Thu Feb 28 00:38:13
2013
@@ -120,7 +120,7 @@ class TestUtilsMixin:
             ld = 'ACCUMULO_LOG_DIR=%s/logs/%s' % (ACCUMULO_HOME, ID)
             execcmd = ['ssh', '-q', host, cp, jo, go, ld] + quote(cmd)
             log.debug(repr(execcmd))
-            return Popen(execcmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, **opts)
+            return Popen(execcmd, stdout=PIPE, stderr=PIPE, **opts)
             
     def shell(self, host, input, **opts):
         """Run accumulo shell with the given input,
@@ -220,7 +220,7 @@ class TestUtilsMixin:
         args = ''
         if timestamp:
             args += "-ts %ld " % int(timestamp)
-        args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1
--createTable ' % (INSTANCE_NAME, ROOT, size, count, start)
+        args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1
--createTable -p %s' % (INSTANCE_NAME, ROOT, size, count, start, ROOT_PASSWORD)
         if colf:
            args = '--columnFamily %s ' % colf + args
         return self.runClassOn(host, klass, args.split(), **kwargs)
@@ -230,7 +230,7 @@ class TestUtilsMixin:
         args = ''
         if timestamp:
             args += "-ts %ld " % int(timestamp)
-        args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 '
% (INSTANCE_NAME, ROOT, size, colf, count, start)
+        args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 -p
%s' % (INSTANCE_NAME, ROOT, size, colf, count, start, ROOT_PASSWORD)
         return self.runClassOn(host, klass, args.split())
 
     def stop_accumulo(self, signal=signal.SIGHUP):

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py Thu Feb 28
00:38:13 2013
@@ -36,7 +36,7 @@ class BinaryTest(unittest.TestCase, Test
         handle = self.runClassOn(self.masterHost(), testClass, 
                                  ['-u', 'root',
                                   '--mode', mode, '-t', table, 
-                                  '--start', min, '--count', max])
+                                  '--start', min, '--count', max, '-p', 'secret'])
         self.waitForStop(handle, 200)
         
     def tearDown(self):

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py Thu Feb 28
00:38:13 2013
@@ -63,7 +63,7 @@ class Examples(TestUtilsMixin, unittest.
 	self.comment("Testing MaxMutation constraint")
 	self.ashell('createtable test_ingest\n'
                     'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n')
-        handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest',
'-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000'])
+        handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest',
'-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD])
         out, err = handle.communicate()
         self.failIf(handle.returncode==0)
         self.failUnless(err.find("MutationsRejectedException: # constraint violations : 1")
>= 0, "Was able to insert a mutation larger than max size")

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py Thu Feb
28 00:38:13 2013
@@ -20,7 +20,7 @@ import unittest
 import time
 import sys
 
-from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME
+from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME, ROOT, ROOT_PASSWORD
 
 log = logging.getLogger('test.auto')
 
@@ -82,12 +82,12 @@ class MultiTableTest(SunnyDayTest):
     
     def ingest(self, host, count, *args, **kwargs):
         klass = 'org.apache.accumulo.test.TestMultiTableIngest'
-        args = '--count %d -u root -i %s ' % (count, INSTANCE_NAME)
+        args = '--count %d -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD)
         return self.runClassOn(host, klass, args.split())
 
     def verify(self, host, count, *args, **kwargs):
         klass = 'org.apache.accumulo.test.TestMultiTableIngest'
-        args = '--count %d --readonly -u root -i %s ' % (count, INSTANCE_NAME)
+        args = '--count %d --readonly -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD)
         return self.runClassOn(host, klass, args.split())
 
 

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py Thu Feb
28 00:38:13 2013
@@ -19,7 +19,7 @@ import logging
 import unittest
 import time
 
-from TestUtils import TestUtilsMixin
+from TestUtils import TestUtilsMixin, ROOT_PASSWORD
 
 log = logging.getLogger('test.auto')
 
@@ -33,7 +33,7 @@ class WriteLots(unittest.TestCase, TestU
 
     def ingest(self, host, start, count, **kwargs):
         klass = 'org.apache.accumulo.test.TestIngest'
-        args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1' % (count,
start)
+        args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1 -p %s' %
(count, start, ROOT_PASSWORD)
         return self.runClassOn(host, klass, args.split(), **kwargs)
 
     def setUp(self):

Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl Thu Feb 28
00:38:13 2013
@@ -19,15 +19,38 @@
 use POSIX qw(strftime);
 
 if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
-	print "Usage : agitator.pl <sleep before kill in minutes> <sleep before tup in
minutes> [<min kill> <max kill>]\n";
+	print "Usage : agitator.pl <min sleep before kill in minutes>[:max sleep before kill
in minutes] <min sleep before tup in minutes>[:max sleep before tup in minutes] [<min
kill> <max kill>]\n";
 	exit(1);
 }
 
 $ACCUMULO_HOME="../../..";
 $HADOOP_PREFIX=$ENV{"HADOOP_PREFIX"};
 
-$sleep1 = $ARGV[0];
-$sleep2 = $ARGV[1];
+@sleeprange1 = split(/:/, $ARGV[0]);
+$sleep1 = $sleeprange1[0];
+
+@sleeprange2 = split(/:/, $ARGV[1]);
+$sleep2 = $sleeprange2[0];
+
+if(scalar(@sleeprange1) > 1){
+    $sleep1max = $sleeprange1[1] + 1;
+}else{
+    $sleep1max = $sleep1;
+}
+
+if($sleep1 > $sleep1max){
+	die("sleep1 > sleep1max $sleep1 > $sleep1max");
+}
+
+if(scalar(@sleeprange2) > 1){
+    $sleep2max = $sleeprange2[1] + 1;
+}else{
+    $sleep2max = $sleep2;
+}
+
+if($sleep2 > $sleep2max){
+	die("sleep2 > sleep2max $sleep2 > $sleep2max");
+}
 
 if(scalar(@ARGV) == 4){
 	$minKill = $ARGV[2];
@@ -104,13 +127,15 @@ while(1){
 		}
 	}
 
-	sleep($sleep2 * 60);
+  $nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
+	sleep($nextsleep2 * 60);
 	$t = strftime "%Y%m%d %H:%M:%S", localtime;
 	print STDERR "$t Running tup\n";
 	system("$ACCUMULO_HOME/bin/tup.sh");
 	print STDERR "$t Running start-dfs\n";
 	system("$HADOOP_PREFIX/bin/start-dfs.sh");
 
-	sleep($sleep1 * 60);
+  $nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
+  sleep($nextsleep1 * 60);
 }
 



Mime
View raw message