Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@accumulo.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: mwalch@apache.org
To: commits@accumulo.apache.org
Message-Id: <f0f1741ec9dd4bd1bdbebc733441f6b9@git.apache.org>
Subject: accumulo-testing git commit: ACCUMULO-4581 Fixing several bugs in
 agitator scripts
Date: Fri, 17 Feb 2017 21:58:39 +0000 (UTC)
archived-at: Fri, 17 Feb 2017 21:58:41 -0000

Repository: accumulo-testing
Updated Branches:
  refs/heads/master 28ca75a36 -> 7495ba604


ACCUMULO-4581 Fixing several bugs in agitator scripts

* Updated how perl scripts are called in bash
* Removed analyze-missing.pl as it relies on centralized logging
* Renamed accumulo home and conf dir variables names in agitator
  perl scripts


Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/7495ba60
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/7495ba60
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/7495ba60

Branch: refs/heads/master
Commit: 7495ba6048ead216e578e8b6cdcc86d4ae8b1b47
Parents: 28ca75a
Author: Mike Walch <mwalch@apache.org>
Authored: Wed Feb 1 15:53:47 2017 -0500
Committer: Mike Walch <mwalch@apache.org>
Committed: Mon Feb 6 15:55:26 2017 -0500

----------------------------------------------------------------------
 bin/accumulo-testing                 |  28 ++++---
 conf/accumulo-testing-env.sh.example |  12 ++-
 libexec/analyze-missing.pl           | 127 ------------------------------
 libexec/datanode-agitator.pl         |  18 ++---
 libexec/master-agitator.pl           |  25 +++---
 libexec/tserver-agitator.pl          |  22 +++---
 6 files changed, 49 insertions(+), 183 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index 7cf7206..84a1d40 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -116,6 +116,8 @@ function determine_mapred_main() {
 }
 
 function start_agitator() {
+  hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be installed. Aborting."; exit 1; }
+
   mkdir -p "${at_home}/logs"
   log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
   libexec="${at_home}/libexec"
@@ -132,17 +134,23 @@ function start_agitator() {
     su -c "$master_cmd >${master_log}.out 2>${master_log}.err" & -m - "$AGTR_ACCUMULO_USER"
     su -c "$tserver_cmd >${tserver_log}.out 2>${tserver_log}.err" & -m - "$AGTR_ACCUMULO_USER"
     su -c "$datanode_cmd >${datanode_log}.out 2>${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
-  elif [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
-    echo "Running master-agitator and tserver-agitator as $AGITATOR_USER Running datanode-agitator as $AGTR_HDFS_USER using sudo."
-    $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
-    $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
-    sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" &
   else
-    echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo. Running datanode-agitator as $AGTR_HDFS_USER using sudo."
-    # Not root, and not the accumulo user, hope you can sudo to it
-    sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> "${master_log}.err" &
-    sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> "${tserver_log}.err" &
-    sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
+    if [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
+      echo "Running master-agitator and tserver-agitator as $AGITATOR_USER"
+      $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+      $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+    else 
+      echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo."
+      sudo -u "$AGTR_ACCUMULO_USER" $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+      sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+    fi
+    if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then
+      echo "Running datanode-agitator as $AGITATOR_USER"
+      $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
+    else
+      echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo."
+      sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
+    fi
   fi
 
   if ${AGTR_HDFS:-false} ; then

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 3f71463..e03c973 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -13,20 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Hadoop
-# ======
+# General
+# =======
 test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
-test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
-
-# Accumulo
-# ========
-# Set the Accumulo version that should be included in the shaded jar
+# Versions set below will be what is included in the shaded jar
 export ACCUMULO_VERSION=`accumulo version`
 export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
 export ZOOKEEPER_VERSION=3.4.9
 
 # Agitator
 # ========
+# Agitator needs know where Accumulo is installed
+test -z "$ACCUMULO_HOME" && export ACCUMULO_HOME=/path/to/accumulo
 # Accumulo user
 AGTR_ACCUMULO_USER=$(whoami)
 # Time (in minutes) between killing Accumulo masters

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/analyze-missing.pl
----------------------------------------------------------------------
diff --git a/libexec/analyze-missing.pl b/libexec/analyze-missing.pl
deleted file mode 100755
index 5cce1b1..0000000
--- a/libexec/analyze-missing.pl
+++ /dev/null
@@ -1,127 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-
-if(scalar(@ARGV) != 4){
-	print "Usage : analyze-missing.pl <accumulo home> <continuous log dir> <user> <pass> \n";
-	exit(1);
-}
-
-$ACCUMULO_HOME=$ARGV[0];
-$CONTINUOUS_LOG_DIR=$ARGV[1];
-$USER=$ARGV[2];
-$PASS=$ARGV[3];
-
-
-@missing = `grep MIS $CONTINUOUS_LOG_DIR/*.err`;
-
-
-
-for $miss (@missing) {
-	chomp($miss);
-	($file, $type, $time, $row) = split(/[: ]/, $miss);
-
-	substr($file, -3, 3, "out");
-
-	$prevRowLine = `grep -B 1 $row $file | grep SRQ | grep -v $row`;
-
-	@prla = split(/\s+/, $prevRowLine);
-	$prevRow = $prla[2];
-#	print $prevRow."\n";
-
-	$aScript = `mktemp /tmp/miss_script.XXXXXXXXXX`;
-	chomp($aScript);
-	open(AS, ">$aScript") || die;
-
-	print AS "table ci\n";
-	print AS "scan -b $prevRow -e $prevRow\n";
-	print AS "scan -b $row -e $row\n";
-	print AS "quit\n";
-	close(AS);
-
-	$exist = 0;
-	$ingestIDSame = 0;
-	$ingestId = "";
-	$count = 0;
-
-	@entries = `$ACCUMULO_HOME/bin/accumulo shell -u $USER -p $PASS -f $aScript | grep $row`;
-	system("rm $aScript");
-
-	for $entry (@entries){
-		chomp($entry);
-		@entryA = split(/[: ]+/, $entry);
-		if($entryA[0] eq $row){
-			$exist = 1;
-
-			if($entryA[4] eq $ingestId){
-				$ingestIDSame = 1;
-			}
-		}else{
-			$ingestId = $entryA[4];
-			$count = hex($entryA[5]);
-		}
-	}
-
-
-	#look in ingest logs
-	@ingestLogs = `ls  $CONTINUOUS_LOG_DIR/*ingest*.out`;
-	@flushTimes = ();
-	chomp(@ingestLogs);
-	for $ingestLog (@ingestLogs){
-		open(IL, "<$ingestLog") || die;
-		
-
-		while($firstLine = <IL>){
-			chomp($firstLine);
-			if($firstLine =~ /UUID.*/){
-				last;
-			}
-		}
-
-		@iinfo = split(/\s+/,$firstLine);
-		if($iinfo[2] eq $ingestId){
-			while($line = <IL>){
-				if($line =~ /FLUSH (\d+) \d+ \d+ (\d+) \d+/){
-					push(@flushTimes, $1);
-					if(scalar(@flushTimes) > 3){
-						shift(@flushTimes);
-					}
-					if($count < $2){
-						last;
-					}
-				}
-			}
-		}
-		
-		
-
-		close(IL);
-	
-		if(scalar(@flushTimes) > 0){
-			last;
-		}
-	} 
-
-	$its0 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[0]/1000);
-	$its1 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[1]/1000);
-	$mts = strftime "%m/%d/%Y_%H:%M:%S", gmtime($time/1000);
-
-	print "$row $exist $ingestIDSame $prevRow $ingestId   $its0   $its1   $mts\n";
-}
-

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/datanode-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/datanode-agitator.pl b/libexec/datanode-agitator.pl
index a98bb66..bb0b38e 100755
--- a/libexec/datanode-agitator.pl
+++ b/libexec/datanode-agitator.pl
@@ -24,16 +24,16 @@ if(scalar(@ARGV) != 5 && scalar(@ARGV) != 3){
   exit(1);
 }
 
-my $ACCUMULO_HOME;
+my $accumuloHome;
 if( defined $ENV{'ACCUMULO_HOME'} ){
-  $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+  $accumuloHome = $ENV{'ACCUMULO_HOME'};
 } else {
-  $cwd=Cwd::cwd();
-  $ACCUMULO_HOME=$cwd . '/../../..';
+  print "ERROR: ACCUMULO_HOME needs to be set!";
+  exit(1);
 }
 $HADOOP_PREFIX=$ARGV[2];
 
-print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
+print "ACCUMULO_HOME=$accumuloHome\n";
 print "HADOOP_PREFIX=$HADOOP_PREFIX\n";
 
 @sleeprange1 = split(/:/, $ARGV[0]);
@@ -62,11 +62,7 @@ if($sleep2 > $sleep2max){
   die("sleep2 > sleep2max $sleep2 > $sleep2max");
 }
 
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
-  $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
-  $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
 
 if(scalar(@ARGV) == 5){
   $minKill = $ARGV[3];
@@ -80,7 +76,7 @@ if($minKill > $maxKill){
   die("minKill > maxKill $minKill > $maxKill");
 }
 
-@tserversRaw = `cat $ACCUMULO_CONF_DIR/tservers`;
+@tserversRaw = `cat $accumuloConfDir/tservers`;
 chomp(@tserversRaw);
 
 for $tserver (@tserversRaw){

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/master-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/master-agitator.pl b/libexec/master-agitator.pl
index d87f17e..949cfbb 100755
--- a/libexec/master-agitator.pl
+++ b/libexec/master-agitator.pl
@@ -15,7 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 use POSIX qw(strftime);
 use Cwd qw();
 
@@ -24,24 +23,20 @@ if(scalar(@ARGV) != 2){
 	exit(1);
 }
 
-my $ACCUMULO_HOME;
+my $accumuloHome;
 if( defined $ENV{'ACCUMULO_HOME'} ){
-  $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+  $accumuloHome = $ENV{'ACCUMULO_HOME'};
 } else {
-  $cwd=Cwd::cwd();
-  $ACCUMULO_HOME=$cwd . '/../../..';
+  print "ERROR: ACCUMULO_HOME needs to be set!";
+  exit(1);
 }
 
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
-        $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
-	$ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
 
 $sleep1 = $ARGV[0];
 $sleep2 = $ARGV[1];
 
-@mastersRaw = `cat $ACCUMULO_CONF_DIR/masters`;
+@mastersRaw = `cat $accumuloConfDir/masters`;
 chomp(@mastersRaw);
 
 for $master (@mastersRaw){
@@ -64,18 +59,18 @@ while(1){
 		system($cmd);
 	}else{
 		print STDERR "$t Killing all masters\n";
-		$cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
+		$cmd = "pssh -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
 		print "$t $cmd\n";
 		system($cmd);
 
 		$file = '';
-		if (-e "$ACCUMULO_CONF_DIR/gc") {
+		if (-e "$accumuloConfDir/gc") {
 			$file = 'gc';
 		} else {
 			$file = 'masters';
 		}
 
-		$cmd = "pssh -h $ACCUMULO_CONF_DIR/$file \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
+		$cmd = "pssh -h $accumuloConfDir/$file \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
 		print "$t $cmd\n";
 		system($cmd);
 	}
@@ -84,7 +79,7 @@ while(1){
 	$t = strftime "%Y%m%d %H:%M:%S", localtime;
 	print STDERR "$t Running start-all\n";
 
-	$cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"$ACCUMULO_HOME/bin/accumulo-service master start\" < /dev/null";
+	$cmd = "pssh -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
 	print "$t $cmd\n";
 	system($cmd);
 }

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/tserver-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/tserver-agitator.pl b/libexec/tserver-agitator.pl
index de29e3a..160824e 100755
--- a/libexec/tserver-agitator.pl
+++ b/libexec/tserver-agitator.pl
@@ -24,15 +24,15 @@ if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
   exit(1);
 }
 
-my $ACCUMULO_HOME;
+my $accumuloHome;
 if( defined $ENV{'ACCUMULO_HOME'} ){
-  $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+  $accumuloHome = $ENV{'ACCUMULO_HOME'};
 } else {
-  $cwd=Cwd::cwd();
-  $ACCUMULO_HOME=$cwd . '/../../..';
+  print "ERROR: ACCUMULO_HOME needs to be set!";
+  exit(1);
 }
 
-print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
+print "ACCUMULO_HOME=$accumuloHome\n";
 
 @sleeprange1 = split(/:/, $ARGV[0]);
 $sleep1 = $sleeprange1[0];
@@ -60,11 +60,7 @@ if($sleep2 > $sleep2max){
   die("sleep2 > sleep2max $sleep2 > $sleep2max");
 }
 
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
-  $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
-  $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
 
 if(scalar(@ARGV) == 4){
   $minKill = $ARGV[2];
@@ -78,7 +74,7 @@ if($minKill > $maxKill){
   die("minKill > maxKill $minKill > $maxKill");
 }
 
-@tserversRaw = `cat $ACCUMULO_CONF_DIR/tservers`;
+@tserversRaw = `cat $accumuloConfDir/tservers`;
 chomp(@tserversRaw);
 
 for $tserver (@tserversRaw){
@@ -118,7 +114,7 @@ while(1){
 
     print STDERR "$t Killing tserver on $server\n";
     # We're the accumulo user, just run the commandj
-    system("ssh $server '$ACCUMULO_HOME/bin/accumulo-service tserver stop'");
+    system("ssh $server '$accumuloHome/bin/accumulo-service tserver kill'");
   }
 
   $nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
@@ -126,7 +122,7 @@ while(1){
   $t = strftime "%Y%m%d %H:%M:%S", localtime;
   print STDERR "$t Running tup\n";
   # restart the as them as the accumulo user
-  system("$ACCUMULO_HOME/libexec/cluster.sh start-tservers");
+  system("$accumuloHome/bin/accumulo-cluster start-tservers");
 
   $nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
   sleep($nextsleep1 * 60);