flume-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mardan Khan <mardan8...@gmail.com>
Subject Re: Installation of full flume on window OS
Date Mon, 30 Jul 2012 13:53:03 GMT
Thanks Inder,

when I am running the $echo $CLASSPATH, I am just getting . (dot) and
nothing more.

Please see below the flume-ng files. Can you just pointout where I should
put echo$CLASSPATH.

Thanks


CLEAN_FLAG=1
################################
# functions
################################

info() {
  if [ ${CLEAN_FLAG} -ne 0 ]; then
    local msg=$1
    echo "Info: $msg" >&2
  fi
}

warn() {
  if [ ${CLEAN_FLAG} -ne 0 ]; then
    local msg=$1
    echo "Warning: $msg" >&2
  fi
}

error() {
  local msg=$1
  local exit_code=$2

  echo "Error: $msg" >&2

  if [ -n "$exit_code" ] ; then
    exit $exit_code
  fi
}

# If avail, add Hadoop paths to the FLUME_CLASSPATH and to the
# FLUME_JAVA_LIBRARY_PATH env vars.
# Requires Flume jars to already be on FLUME_CLASSPATH.
add_hadoop_paths() {
  local HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" \
      which hadoop 2>/dev/null)

  if [ -f "${HADOOP_IN_PATH}" ]; then
    info "Including Hadoop libraries found via ($HADOOP_IN_PATH) for HDFS
access"

    # determine hadoop java.library.path and use that for flume
    local HADOOP_CLASSPATH=""
    local HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$FLUME_CLASSPATH" \
        ${HADOOP_IN_PATH} org.apache.flume.tools.GetJavaProperty \
        java.library.path 2>/dev/null)

    # look for the line that has the desired property value
    # (considering extraneous output from some GC options that write to
stdout)
    # IFS = InternalFieldSeparator (set to recognize only newline char as
delimiter)
    IFS=$'\n'
    for line in $HADOOP_JAVA_LIBRARY_PATH; do
      if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
        HADOOP_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
        break
      fi
    done
    unset IFS

    if [ -n "${HADOOP_JAVA_LIBRARY_PATH}" ]; then

FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HADOOP_JAVA_LIBRARY_PATH"
    fi

    # determine hadoop classpath
    HADOOP_CLASSPATH=$($HADOOP_IN_PATH classpath 2>/dev/null)

    # hack up and filter hadoop classpath
    local ELEMENTS=$(sed -e 's/:/ /g' <<<${HADOOP_CLASSPATH})
    local ELEMENT
    for ELEMENT in $ELEMENTS; do
      local PIECE
      for PIECE in $(echo $ELEMENT); do
        if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
          info "Excluding $PIECE from classpath"
          continue
        else
          FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
        fi
      done
    done

  fi
}
add_HBASE_paths() {
  local HBASE_IN_PATH=$(PATH="${HBASE_HOME}/bin:$PATH" \
      which hbase 2>/dev/null)

  if [ -f "${HBASE_IN_PATH}" ]; then
    info "Including HBASE libraries found via ($HBASE_IN_PATH) for HBASE
access"

    # determine HBASE java.library.path and use that for flume
    local HBASE_CLASSPATH=""
    local HBASE_JAVA_LIBRARY_PATH=$(HBASE_CLASSPATH="$FLUME_CLASSPATH" \
        ${HBASE_IN_PATH} org.apache.flume.tools.GetJavaProperty \
        java.library.path 2>/dev/null)

    # look for the line that has the desired property value
    # (considering extraneous output from some GC options that write to
stdout)
    # IFS = InternalFieldSeparator (set to recognize only newline char as
delimiter)
    IFS=$'\n'
    for line in $HBASE_JAVA_LIBRARY_PATH; do
      if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
        HBASE_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
        break
      fi
    done
    unset IFS

    if [ -n "${HBASE_JAVA_LIBRARY_PATH}" ]; then

FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HBASE_JAVA_LIBRARY_PATH"
    fi

    # determine HBASE classpath
    HBASE_CLASSPATH=$($HBASE_IN_PATH classpath 2>/dev/null)

    # hack up and filter HBASE classpath
    local ELEMENTS=$(sed -e 's/:/ /g' <<<${HBASE_CLASSPATH})
    local ELEMENT
    for ELEMENT in $ELEMENTS; do
      local PIECE
      for PIECE in $(echo $ELEMENT); do
        if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
          info "Excluding $PIECE from classpath"
          continue
        else
          FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
        fi
      done
    done
    FLUME_CLASSPATH="$FLUME_CLASSPATH:$HBASE_HOME/conf"

  fi
}

display_help() {
  cat <<EOF
Usage: $0 <command> [options]...

commands:
  help                  display this help text
  agent                 run a Flume agent
  avro-client           run an avro Flume client
  version               show Flume version info

global options:
  --conf,-c <conf>      use configs in <conf> directory
  --classpath,-C <cp>   append to the classpath
  --dryrun,-d           do not actually start Flume, just print the command
  -Dproperty=value      sets a JDK system property value

agent options:
  --conf-file,-f <file> specify a config file (required)
  --name,-n <name>      the name of this agent (required)
  --help,-h             display help text

avro-client options:
  --host,-H <host>      hostname to which events will be sent (required)
  --port,-p <port>      port of the avro source (required)
  --filename,-F <file>  text file to stream to avro source [default: std
input]
  --headerFile,-R <file> headerFile containing headers as key/value pairs
on each new line
  --help,-h             display help text

Note that if <conf> directory is specified, then it is always included first
in the classpath.

EOF
}

run_flume() {
  local FLUME_APPLICATION_CLASS

  if [ "$#" -gt 0 ]; then
    FLUME_APPLICATION_CLASS=$1
    shift
  else
    error "Must specify flume application class" 1
  fi

  if [ ${CLEAN_FLAG} -ne 0 ]; then
    set -x
  fi
  $EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
      -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH
"$FLUME_APPLICATION_CLASS" $*
}

################################
# main
################################

# set default params
FLUME_CLASSPATH=""
FLUME_JAVA_LIBRARY_PATH=""
JAVA_OPTS="-Xmx20m"

opt_conf=""
opt_classpath=""
opt_java_props=""
opt_dryrun=""

mode=$1
shift

case "$mode" in
  help)
    display_help
    exit 0
    ;;
  agent)
    opt_agent=1
    ;;
  node)
    opt_agent=1
    warn "The \"node\" command is deprecated. Please use \"agent\" instead."
    ;;
  avro-client)
    opt_avro_client=1
    ;;
  version)
   opt_version=1
   CLEAN_FLAG=0
   ;;
  *)
    error "Unknown or unspecified command '$mode'"
    echo
    display_help
    exit 1
    ;;
esac

while [ -n "$*" ] ; do
  arg=$1
  shift

  case "$arg" in
    --conf|-c)
      [ -n "$1" ] || error "Option --conf requires an argument" 1
      opt_conf=$1
      shift
      ;;
    --classpath|-C)
      [ -n "$1" ] || error "Option --classpath requires an argument" 1
      opt_classpath=$1
      shift
      ;;
    --dryrun|-d)
      opt_dryrun="1"
      ;;
    -D*)
      opt_java_props="$opt_java_props $arg"
      ;;
    *)
      args="$args $arg"
      ;;
  esac
done

# make opt_conf absolute
if [[ -n "$opt_conf" && -d "$opt_conf" ]]; then
  opt_conf=$(cd $opt_conf; pwd)
fi

# allow users to override the default env vars via conf/flume-env.sh
if [ -z "$opt_conf" ]; then
  warn "No configuration directory set! Use --conf <dir> to override."
elif [ -f "$opt_conf/flume-env.sh" ]; then
  info "Sourcing environment configuration script $opt_conf/flume-env.sh"
  source "$opt_conf/flume-env.sh"
fi

# append command-line java options to stock or env script JAVA_OPTS
if [ -n "${opt_java_props}" ]; then
  JAVA_OPTS="${JAVA_OPTS} ${opt_java_props}"
fi

# prepend command-line classpath to env script classpath
if [ -n "${opt_classpath}" ]; then
  if [ -n "${FLUME_CLASSPATH}" ]; then
    FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"
  else
    FLUME_CLASSPATH="${opt_classpath}"
  fi
fi

if [ -z "${FLUME_HOME}" ]; then
  FLUME_HOME=$(cd $(dirname $0)/..; pwd)
fi

# prepend $FLUME_HOME/lib jars to the specified classpath (if any)
if [ -n "${FLUME_CLASSPATH}" ] ; then
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
else
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
fi

# find java
if [ -z "${JAVA_HOME}" ] ; then
  warn "JAVA_HOME is not set!"
  # Try to use Bigtop to autodetect JAVA_HOME if it's available
  if [ -e /usr/libexec/bigtop-detect-javahome ] ; then
    . /usr/libexec/bigtop-detect-javahome
  elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ] ; then
    . /usr/lib/bigtop-utils/bigtop-detect-javahome
  fi

  # Using java from path if bigtop is not installed or couldn't find it
  if [ -z "${JAVA_HOME}" ] ; then
    JAVA_DEFAULT=$(type -p java)
    [ -n "$JAVA_DEFAULT" ] || error "Unable to find java executable. Is it
in your PATH?" 1
    JAVA_HOME=$(cd $(dirname $JAVA_DEFAULT)/..; pwd)
  fi
fi

# look for hadoop libs
add_hadoop_paths
add_HBASE_paths

# prepend conf dir to classpath
if [ -n "$opt_conf" ]; then
  FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
fi

# allow dryrun
EXEC="exec"
if [ -n "${opt_dryrun}" ]; then
  warn "Dryrun mode enabled (will not actually initiate startup)"
  EXEC="echo"
fi

# finally, invoke the appropriate command
if [ -n "$opt_agent" ] ; then
  run_flume $FLUME_AGENT_CLASS $args
elif [ -n "$opt_avro_client" ] ; then
  run_flume $FLUME_AVRO_CLIENT_CLASS $args
elif [ -n "${opt_version}" ] ; then
  run_flume $FLUME_VERSION_CLASS $args
else
  error "This message should never appear" 1
fi

exit 0







On Mon, Jul 30, 2012 at 2:40 PM, Inder Pall <inder.pall@gmail.com> wrote:

> Hey if you edit flume-ng script and just before java is invoked put an
>
> echo $CLASSPATH that should help in figuring what all dir's are there in
> CLASSPATH.
> I am sorry but i am a big fan of windows hence don't run that O.S. to try
> it out.
>
> Thanks,
> - inder
>
> On Mon, Jul 30, 2012 at 7:00 PM, mardan Khan <mardan8310@gmail.com> wrote:
>
>> HI Inder,
>>
>> Sorry for the previous email. Actually I past the snapshot directly into
>> editor which convert into some noisy text. Please see the snapshot as
>> attachment .
>>
>> Please let me know for any solution
>>
>> Thanks
>>
>>
>>
>>
>> On Mon, Jul 30, 2012 at 1:20 PM, Inder Pall <inder.pall@gmail.com> wrote:
>>
>>> can you echo your CLASSPATH in the flume startup script.
>>>
>>> - inder
>>>
>>>
>>> On Mon, Jul 30, 2012 at 4:25 PM, mardan Khan <mardan8310@gmail.com>wrote:
>>>
>>>> Hi Brock
>>>>
>>>> Thanks for reply. I have solved the problem of cygwin. Now i am getting
>>>> another error message: Could not find or load main class of
>>>> org.apache.flume.node.Application.
>>>>
>>>> I am running the following command:
>>>>
>>>> bin/flume-ng agent -n agent -c conf -f
>>>> conf/flume-conf.properties.template
>>>>
>>>> Thanks
>>>>
>>>>
>>>>
>>>> On Sun, Jul 29, 2012 at 6:03 PM, Brock Noland <brock@cloudera.com>wrote:
>>>>
>>>>> Hi,
>>>>>
>>>>> Unfortunately I am not familiar with cygwin and don't have a windows
>>>>> computer to test on. Hopefully someone else on the list has an insight.
>>>>>
>>>>> Brock
>>>>>
>>>>>
>>>>> On Sun, Jul 29, 2012 at 11:27 AM, mardan Khan <mardan8310@gmail.com>wrote:
>>>>>
>>>>>> Thanks Brock.
>>>>>>
>>>>>>
>>>>>> Now when i am run the command:  flume$ bin/flume-ng agent -n agent
-f
>>>>>> conf/flume-conf.properties.
>>>>>> template.
>>>>>>
>>>>>>
>>>>>> Cygwin Warning: MS-DOS style path detected
>>>>>> /usr/local/bin/c:\program......................................................................
>>>>>> bin/flume-ng line:210 exec c:\program not found.
>>>>>>
>>>>>> How to solve this problem.
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> On Sun, Jul 29, 2012 at 4:41 PM, Brock Noland <brock@cloudera.com>wrote:
>>>>>>
>>>>>>> Hi,
>>>>>>>
>>>>>>> Inline
>>>>>>>
>>>>>>> On Sat, Jul 28, 2012 at 11:54 PM, mardan Khan <mardan8310@gmail.com>wrote:
>>>>>>>
>>>>>>>> Hi Brock.
>>>>>>>>
>>>>>>>> Thanks for reply. I want to use the full flume as i have
to upload
>>>>>>>> the data into hdfs. I have installed the cygwin and hadoop.
>>>>>>>>
>>>>>>>
>>>>>>> OK this is a great start.
>>>>>>>
>>>>>>>
>>>>>>>> I have download the flume binary from the mentioned link.
Now you
>>>>>>>> please let me explain the follow:
>>>>>>>>
>>>>>>>> 1). I need just to decopress the downloaded folder?
>>>>>>>>
>>>>>>>
>>>>>>> Yes you download a tar.gz which is like a zip. In cygwin you
can use
>>>>>>> the command tar -zxvf to untar it.
>>>>>>>
>>>>>>>
>>>>>>>> 2). where i should decompress the flume binary. Should i
decompress
>>>>>>>> c:\cygwin\home\user-name
>>>>>>>>
>>>>>>>
>>>>>>> Whereever you want, however a common place for "tarball" software
is
>>>>>>> /usr/local or /opt
>>>>>>>
>>>>>>>
>>>>>>>> 3). How I will run the flume-ng.
>>>>>>>>
>>>>>>>
>>>>>>> You configure flume and then run the flume-ng command in the
bin
>>>>>>> directory
>>>>>>>
>>>>>>>
>>>>>>>>  4) The flume.conf file will configure same as configuring
using
>>>>>>>> Linux?
>>>>>>>>
>>>>>>>>
>>>>>>> Yes configure flume will be the same as on Linux in cygwin.
>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> Many thanks
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> On Sun, Jul 29, 2012 at 4:20 AM, Brock Noland <brock@cloudera.com>wrote:
>>>>>>>>
>>>>>>>>> Hi,
>>>>>>>>>
>>>>>>>>> If you just want run a flume agent on Windows you just
download
>>>>>>>>> the binary here: http://flume.apache.org/download.html
>>>>>>>>>
>>>>>>>>> Note that the flume startup script is written in bash
so unless
>>>>>>>>> you use cygwin you'll need to start the JVM and setup
the classpath
>>>>>>>>> yourself.
>>>>>>>>>
>>>>>>>>> Brock
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> On Sat, Jul 28, 2012 at 8:01 PM, mardan Khan <mardan8310@gmail.com
>>>>>>>>> > wrote:
>>>>>>>>>
>>>>>>>>>> Hi,
>>>>>>>>>>
>>>>>>>>>> I am struggling from couple of weeks to install and
configure the
>>>>>>>>>> flume1.x on windows but could not success. Could
you please someone show me
>>>>>>>>>> step-by-step how to install flume on window. I have
read a couple of posts
>>>>>>>>>> which mentioned the git and Maven but i dont understand
how can use it to
>>>>>>>>>> install the flume.
>>>>>>>>>>
>>>>>>>>>> Please help me as i am total stuck and running out
of time.
>>>>>>>>>>
>>>>>>>>>> Thanks
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> --
>>>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>
>>>>>>
>>>>>>
>>>>>
>>>>>
>>>>> --
>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>> http://incubator.apache.org/mrunit/
>>>>>
>>>>
>>>>
>>>
>>>
>>> --
>>> Thanks,
>>> - Inder
>>>   Tech Platforms @Inmobi
>>>   Linkedin - http://goo.gl/eR4Ub
>>>
>>
>>
>
>
> --
> Thanks,
> - Inder
>   Tech Platforms @Inmobi
>   Linkedin - http://goo.gl/eR4Ub
>

Mime
View raw message