accumulo-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubbsii <...@git.apache.org>
Subject [GitHub] accumulo pull request #160: Simplify Accumulo scripts and config files
Date Wed, 12 Oct 2016 21:39:29 GMT
Github user ctubbsii commented on a diff in the pull request:

    https://github.com/apache/accumulo/pull/160#discussion_r83104635
  
    --- Diff: assemble/bin/accumulo ---
    @@ -51,137 +75,179 @@ locationByProgram()
        eval "${2}=${RESULT}"
     }
     
    -test -z "${JAVA_HOME}"      && locationByProgram java JAVA_HOME
    -test -z "${HADOOP_PREFIX}"  && locationByProgram hadoop HADOOP_PREFIX
    -test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME
    -
    -DEFAULT_GENERAL_JAVA_OPTS=""
    -
    -#
    -# ACCUMULO_XTRAJARS is where all of the commandline -add items go into for reading by
accumulo.
    -# It also holds the JAR run with the jar command and, if possible, any items in the JAR
manifest's Class-Path.
    -#
    -if [ "$1" = "-add" ] ; then
    -    export ACCUMULO_XTRAJARS="$2"
    -    shift 2
    -else
    -    export ACCUMULO_XTRAJARS=""
    -fi
    -if [ "$1" = "jar" -a -f "$2" ] ; then
    -    if [[ $2 =~ ^/ ]]; then
    -      jardir="$(dirname "$2")"
    -      jarfile="$2"
    -    else
    -      jardir="$(pwd)"
    -      jarfile="${jardir}/${2}"
    -    fi
    -    if jar tf "$jarfile" | grep -q META-INF/MANIFEST.MF ; then
    -      cp="$(unzip -p "$jarfile" META-INF/MANIFEST.MF | grep ^Class-Path: | sed 's/^Class-Path:
*//')"
    -      if [[ -n "$cp" ]] ; then
    -         for j in $cp; do
    -            if [[ "$j" != "Class-Path:" ]] ; then
    -               ACCUMULO_XTRAJARS="${jardir}/${j},$ACCUMULO_XTRAJARS"
    -            fi
    -         done
    +function exec_java() {
    +
    +  . "${bin}"/../scripts/config.sh
    +
    +  START_JAR="${ACCUMULO_HOME}/lib/accumulo-start.jar"
    +
    +  test -z "${JAVA_HOME}"      && locationByProgram java JAVA_HOME
    +  test -z "${HADOOP_PREFIX}"  && locationByProgram hadoop HADOOP_PREFIX
    +  test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME
    +
    +  DEFAULT_GENERAL_JAVA_OPTS=""
    +
    +  #
    +  # ACCUMULO_XTRAJARS is where all of the commandline -add items go into for reading
by accumulo.
    +  # It also holds the JAR run with the jar command and, if possible, any items in the
JAR manifest's Class-Path.
    +  #
    +  if [ "$1" = "-add" ] ; then
    +      export ACCUMULO_XTRAJARS="$2"
    +      shift 2
    +  else
    +      export ACCUMULO_XTRAJARS=""
    +  fi
    +  if [ "$1" = "jar" -a -f "$2" ] ; then
    +      if [[ $2 =~ ^/ ]]; then
    +        jardir="$(dirname "$2")"
    +        jarfile="$2"
    +      else
    +        jardir="$(pwd)"
    +        jarfile="${jardir}/${2}"
    +      fi
    +      if jar tf "$jarfile" | grep -q META-INF/MANIFEST.MF ; then
    +        cp="$(unzip -p "$jarfile" META-INF/MANIFEST.MF | grep ^Class-Path: | sed 's/^Class-Path:
*//')"
    +        if [[ -n "$cp" ]] ; then
    +           for j in $cp; do
    +              if [[ "$j" != "Class-Path:" ]] ; then
    +                 ACCUMULO_XTRAJARS="${jardir}/${j},$ACCUMULO_XTRAJARS"
    +              fi
    +           done
    +        fi
           fi
    +      ACCUMULO_XTRAJARS="${jarfile},$ACCUMULO_XTRAJARS"
    +  fi
    +
    +  #
    +  # Set up -D switches for JAAS and Kerberos if env variables set
    +  #
    +  if [[ ! -z ${ACCUMULO_JAAS_CONF} ]]; then
    +    ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.auth.login.config=${ACCUMULO_JAAS_CONF}"
    +  fi
    +
    +  if [[ ! -z ${ACCUMULO_KRB5_CONF} ]]; then
    +    ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.krb5.conf=${ACCUMULO_KRB5_CONF}"
    +  fi
    +
    +  #
    +  # Add appropriate options for process type
    +  #
    +  case "$1" in
    +  master)  export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MASTER_OPTS}" ;;
    +  gc)      export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_GC_OPTS}" ;;
    +  tserver*) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_TSERVER_OPTS}"
;;
    +  monitor) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MONITOR_OPTS}" ;;
    +  shell)   export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_SHELL_OPTS}" ;;
    +  *)       export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_OTHER_OPTS}" ;;
    +  esac
    +
    +  LOG4J_JAR=$(find -H "${HADOOP_PREFIX}/lib" "${HADOOP_PREFIX}"/share/hadoop/common/lib
-name 'log4j*.jar' -print 2>/dev/null | head -1)
    +  SLF4J_JARS="${ACCUMULO_HOME}/lib/slf4j-api.jar:${ACCUMULO_HOME}/lib/slf4j-log4j12.jar"
    +
    +  # The `find` command could fail for environmental reasons or bad configuration
    +  # Avoid trying to run Accumulo when we can't find the jar
    +  if [ -z "${LOG4J_JAR}" -a -z "${CLASSPATH}" ]; then
    +     echo "Could not locate Log4j jar in Hadoop installation at ${HADOOP_PREFIX}"
    +     exit 1
    +  fi
    +
    +  if [[ ! " gc master monitor tserver tracer " =~ " $1 " ]]; then
    +    if [[ -f ${ACCUMULO_CONF_DIR}/log4j.properties ]]; then
    +      export ACCUMULO_OPTS="${ACCUMULO_OPTS} -Dlog4j.configuration=file:${ACCUMULO_CONF_DIR}/log4j.properties"
    +    else
    +      export ACCUMULO_OPTS="${ACCUMULO_OPTS} -Dlog4j.configuration=file:${ACCUMULO_CONF_DIR}/examples/log4j.properties"
         fi
    -    ACCUMULO_XTRAJARS="${jarfile},$ACCUMULO_XTRAJARS"
    -fi
    -
    -#
    -# Set up -D switches for JAAS and Kerberos if env variables set
    -#
    -if [[ ! -z ${ACCUMULO_JAAS_CONF} ]]; then
    -  ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.auth.login.config=${ACCUMULO_JAAS_CONF}"
    -fi
    -
    -if [[ ! -z ${ACCUMULO_KRB5_CONF} ]]; then
    -  ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.krb5.conf=${ACCUMULO_KRB5_CONF}"
    -fi
    +  fi
    +
    +  CLASSPATH="${START_JAR}:${ACCUMULO_CONF_DIR}:${SLF4J_JARS}:${LOG4J_JAR}:${CLASSPATH}"
    +
    +  if [ -z "${JAVA_HOME}" -o ! -d "${JAVA_HOME}" ]; then
    +     echo "JAVA_HOME is not set or is not a directory.  Please make sure it's set globally
or in conf/accumulo-env.sh"
    +     exit 1
    +  fi
    +  if [ -z "${HADOOP_PREFIX}" -o ! -d "${HADOOP_PREFIX}" ]; then
    +     echo "HADOOP_PREFIX is not set or is not a directory.  Please make sure it's set
globally or in conf/accumulo-env.sh"
    +     exit 1
    +  fi
    +  if [ -z "${ZOOKEEPER_HOME}" -o ! -d "${ZOOKEEPER_HOME}" ]; then
    +     echo "ZOOKEEPER_HOME is not set or is not a directory.  Please make sure it's set
globally or in conf/accumulo-env.sh"
    +     exit 1
    +  fi
    +
    +  # This is default for hadoop 2.x;
    +  #   for another distribution, specify (DY)LD_LIBRARY_PATH
    +  #   explicitly in ${ACCUMULO_HOME}/conf/accumulo-env.sh
    +  #   usually something like:
    +  #     ${HADOOP_PREFIX}/lib/native/${PLATFORM}
    +  if [ -e "${HADOOP_PREFIX}/lib/native/libhadoop.so" ]; then
    +     LIB_PATH="${HADOOP_PREFIX}/lib/native"
    +     LD_LIBRARY_PATH="${LIB_PATH}:${LD_LIBRARY_PATH}"     # For Linux
    +     DYLD_LIBRARY_PATH="${LIB_PATH}:${DYLD_LIBRARY_PATH}" # For Mac
    +  fi
    +
    +  # Export the variables just in case they are not exported
    +  # This makes them available to java
    +  export JAVA_HOME HADOOP_PREFIX ZOOKEEPER_HOME LD_LIBRARY_PATH DYLD_LIBRARY_PATH
    +
    +  # Strip the instance from $1
    +  APP=$1
    +  # Avoid setting an instance unless it's necessary to ensure consistency in filenames
    +  INSTANCE=""
    +  # Avoid setting a pointless system property
    +  INSTANCE_OPT=""
    +  if [[ "$1" =~ ^tserver-[1-9][0-9]*$ ]]; then
    +    APP="$(echo "$1" | cut -d'-' -f1)"
    +    # Appending the trailing underscore to make single-tserver deploys look how they
did
    +    INSTANCE="$(echo "$1" | cut -d'-' -f2)_"
    +
    +    #Rewrite the input arguments
    +    set -- "$APP" "${@:2}"
    +
    +    # The extra system property we'll pass to the java cmd
    +    INSTANCE_OPT="-Daccumulo.service.instance=${INSTANCE}"
    +  fi
    +
    +  #
    +  # app isn't used anywhere, but it makes the process easier to spot when ps/top/snmp
truncate the command line
    +  JAVA="${JAVA_HOME}/bin/java"
    +  exec "$JAVA" "-Dapp=$1" \
    +     $INSTANCE_OPT \
    +     $ACCUMULO_OPTS \
    +     -classpath "${CLASSPATH}" \
    +     -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" \
    +     -XX:-OmitStackTraceInFastThrow \
    +     -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
\
    +     -Dorg.apache.accumulo.core.home.dir="${ACCUMULO_HOME}" \
    +     -Dhadoop.home.dir="${HADOOP_PREFIX}" \
    +     -Dzookeeper.home.dir="${ZOOKEEPER_HOME}" \
    +     org.apache.accumulo.start.Main \
    +     "$@"
    +}
     
    -#
    -# Add appropriate options for process type
    -#
    -case "$1" in
    -master)  export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MASTER_OPTS}" ;;
    -gc)      export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_GC_OPTS}" ;;
    -tserver*) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_TSERVER_OPTS}" ;;
    -monitor) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MONITOR_OPTS}" ;;
    -shell)   export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_SHELL_OPTS}" ;;
    -*)       export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_OTHER_OPTS}" ;;
    -esac
    -
    -XML_FILES="${ACCUMULO_CONF_DIR}"
    -LOG4J_JAR=$(find -H "${HADOOP_PREFIX}/lib" "${HADOOP_PREFIX}"/share/hadoop/common/lib
-name 'log4j*.jar' -print 2>/dev/null | head -1)
    -SLF4J_JARS="${ACCUMULO_HOME}/lib/slf4j-api.jar:${ACCUMULO_HOME}/lib/slf4j-log4j12.jar"
    -
    -# The `find` command could fail for environmental reasons or bad configuration
    -# Avoid trying to run Accumulo when we can't find the jar
    -if [ -z "${LOG4J_JAR}" -a -z "${CLASSPATH}" ]; then
    -   echo "Could not locate Log4j jar in Hadoop installation at ${HADOOP_PREFIX}"
    -   exit 1
    -fi
    -
    -CLASSPATH="${XML_FILES}:${START_JAR}:${SLF4J_JARS}:${LOG4J_JAR}:${CLASSPATH}"
    -
    -if [ -z "${JAVA_HOME}" -o ! -d "${JAVA_HOME}" ]; then
    -   echo "JAVA_HOME is not set or is not a directory.  Please make sure it's set globally
or in conf/accumulo-env.sh"
    -   exit 1
    -fi
    -if [ -z "${HADOOP_PREFIX}" -o ! -d "${HADOOP_PREFIX}" ]; then
    -   echo "HADOOP_PREFIX is not set or is not a directory.  Please make sure it's set globally
or in conf/accumulo-env.sh"
    -   exit 1
    -fi
    -if [ -z "${ZOOKEEPER_HOME}" -o ! -d "${ZOOKEEPER_HOME}" ]; then
    -   echo "ZOOKEEPER_HOME is not set or is not a directory.  Please make sure it's set
globally or in conf/accumulo-env.sh"
    -   exit 1
    -fi
    -
    -# This is default for hadoop 2.x;
    -#   for another distribution, specify (DY)LD_LIBRARY_PATH
    -#   explicitly in ${ACCUMULO_HOME}/conf/accumulo-env.sh
    -#   usually something like:
    -#     ${HADOOP_PREFIX}/lib/native/${PLATFORM}
    -if [ -e "${HADOOP_PREFIX}/lib/native/libhadoop.so" ]; then
    -   LIB_PATH="${HADOOP_PREFIX}/lib/native"
    -   LD_LIBRARY_PATH="${LIB_PATH}:${LD_LIBRARY_PATH}"     # For Linux
    -   DYLD_LIBRARY_PATH="${LIB_PATH}:${DYLD_LIBRARY_PATH}" # For Mac
    -fi
    -
    -# Export the variables just in case they are not exported
    -# This makes them available to java
    -export JAVA_HOME HADOOP_PREFIX ZOOKEEPER_HOME LD_LIBRARY_PATH DYLD_LIBRARY_PATH
    -
    -# Strip the instance from $1
    -APP=$1
    -# Avoid setting an instance unless it's necessary to ensure consistency in filenames
    -INSTANCE=""
    -# Avoid setting a pointless system property
    -INSTANCE_OPT=""
    -if [[ "$1" =~ ^tserver-[1-9][0-9]*$ ]]; then
    -  APP="$(echo "$1" | cut -d'-' -f1)"
    -  # Appending the trailing underscore to make single-tserver deploys look how they did
    -  INSTANCE="$(echo "$1" | cut -d'-' -f2)_"
    -
    -  #Rewrite the input arguments
    -  set -- "$APP" "${@:2}"
    -
    -  # The extra system property we'll pass to the java cmd
    -  INSTANCE_OPT="-Daccumulo.service.instance=${INSTANCE}"
    -fi
    +function main() {
    +  scripts=${bin}/../scripts
    +
    +  if [[ -z $1 ]]; then
    +    invalid_args "<command> cannot be empty"
    +  fi
    +
    +  case "$1" in
    +    admin|classpath|create-token|gc|init|info|jar|login-info|master|monitor|shell|tserver|tracer|version)
    +      exec_java $1 ${*:2}
    +      ;;
    +    exec)
    +      exec_java ${*:2}
    +      ;;
    +    bootstrap-config)
    +      "${scripts}/bootstrap-config.sh" ${*:2}
    +      ;;
    +    build-native-lib)
    +      "${scripts}/build-native-library.sh"
    +      ;;
    --- End diff --
    
    Public API or not, though, if breaking this (rather, relocating it and documenting that)
is necessary to put us on a good path for 2.0, I'd prefer we do that. Our scripts our out
of control and trying to do way too much. It makes it harder to extend and package Accumulo
downstream.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message