accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [2/5] accumulo git commit: ACCUMULO-3793 use numctl to turn on memory interleaving
Date Mon, 11 May 2015 18:16:14 GMT
ACCUMULO-3793 use numctl to turn on memory interleaving


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aee2489
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aee2489
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aee2489

Branch: refs/heads/1.7
Commit: 9aee2489b8b824444639223f4d2ac144501d4612
Parents: 1c6e4b6 b821487
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon May 11 14:14:17 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon May 11 14:14:17 2015 -0400

----------------------------------------------------------------------
 assemble/bin/config.sh       | 9 +++++++++
 assemble/bin/start-server.sh | 4 ++--
 2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aee2489/assemble/bin/config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/config.sh
index d9bf8d4,9bee91b..ed393bb
--- a/assemble/bin/config.sh
+++ b/assemble/bin/config.sh
@@@ -104,6 -115,51 +104,15 @@@ the
  fi
  export HADOOP_PREFIX
  
 -unset MASTER1
 -if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then
 -  MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1)
 -fi
 -
 -if [ -z "${MONITOR}" ] ; then
 -  MONITOR=$MASTER1
 -  if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then
 -      MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1)
 -  fi
 -  if [ -z "${MONITOR}" ] ; then
 -    echo "Could not infer a Monitor role. You need to either define the MONITOR env variable,
define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is
non-empty."
 -    exit 1
 -  fi
 -fi
 -if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then
 -  if [ -z "${MASTER1}" ] ; then
 -    echo "Could not find a master node to use as a default for the tracer role. Either set
up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty."
 -    exit 1
 -  else
 -    echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers"
 -  fi
 -
 -fi
 -
 -if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then
 -  if [ -z "${MASTER1}" ] ; then
 -    echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\"
or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty."
 -    exit 1
 -  else
 -    echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc"
 -  fi
 -fi
 -
+ NUMA=`which numactl 2>/dev/null`
+ NUMACTL_EXISTS="$?"
+ NUMACTL_ARGS="--interleave=all"
+ if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then
+   export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}"
+ else
+   export NUMA_CMD=""
+ fi
+ 
 -SSH='ssh -qnf -o ConnectTimeout=2'
 -
  export HADOOP_HOME=$HADOOP_PREFIX
  export HADOOP_HOME_WARN_SUPPRESS=true
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aee2489/assemble/bin/start-server.sh
----------------------------------------------------------------------
diff --cc assemble/bin/start-server.sh
index c52bd64,daa8495..3587ec9
--- a/assemble/bin/start-server.sh
+++ b/assemble/bin/start-server.sh
@@@ -64,28 -66,23 +64,28 @@@ if [[ ${SERVICE} == "monitor" && ${ACCU
      ADDRESS="0.0.0.0"
  fi
  
 -if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then
 +if [[ $HOST == localhost || $HOST == "$(hostname -f)" || $HOST = "$IP" ]]; then
     PID=$(ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE" | grep
-v grep | awk {'print $2'} | head -1)
  else
 -   PID=$($SSH $HOST ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE"
| grep -v grep | awk {'print $2'} | head -1)
 +   PID=$($SSH "$HOST" ps -ef | egrep "${ACCUMULO_HOME}/.*/accumulo.*.jar" | grep "Main $SERVICE"
| grep -v grep | awk {'print $2'} | head -1)
  fi
  
 -if [ -z "$PID" ]; then
 +if [[ -z "$PID" ]]; then
     echo "Starting $LONGNAME on $HOST"
 -   if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then
 -      ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out
2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err & 
 +   COMMAND="${bin}/accumulo"
 +   if [ "${ACCUMULO_WATCHER}" = "true" ]; then
 +      COMMAND="${bin}/accumulo_watcher.sh ${LOGHOST}"
 +   fi
 +
 +   if [[ $HOST == localhost || $HOST == "$(hostname -f)" || $HOST = "$IP" ]]; then
-       "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out"
2>"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" & 
++      ${NUMA_CMD} "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out"
2>"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" & 
        MAX_FILES_OPEN=$(ulimit -n)
     else
-       $SSH "$HOST" "bash -c 'exec nohup $COMMAND ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out
2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
 -      $SSH $HOST "bash -c 'exec nohup ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS}
>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err'
&"
 -      MAX_FILES_OPEN=$($SSH $HOST "/usr/bin/env bash -c 'ulimit -n'") 
++      $SSH "$HOST" "bash -c 'exec nohup ${NUMA_CMD} $COMMAND ${SERVICE} --address ${ADDRESS}
>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err'
&"
 +      MAX_FILES_OPEN=$($SSH "$HOST" "/usr/bin/env bash -c 'ulimit -n'") 
     fi
  
 -   if [ -n "$MAX_FILES_OPEN" ] && [ -n "$SLAVES" ] ; then
 +   if [[ -n $MAX_FILES_OPEN && -n $SLAVES ]] ; then
        MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768}
        if (( SLAVES > 10 )) && (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED ))
        then


Mime
View raw message