flume-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vijay k <k.vija...@gmail.com>
Subject Re: Flume agent failure
Date Wed, 04 Jul 2012 10:05:34 GMT
Hi Mike,

Please find the following commands output:

which hadoop -- hadoop 0.20.2
echo $HADOOP_HOME   --- /hadoop-install/hadoop
hadoop classpath        -- /usr/local/hadoop/bin/hadoop
which java    --1.6
echo $JAVA_HOME    --- /usr/lib/jvm/java-6-sun-1.6.0.26/jre
java -version
 -- java version "1.6.0_26"
Java(TM) SE Runtime Environment (build 1.6.0_26-b03)
Java HotSpot(TM) Server VM (build 20.1-b02, mixed mode)

flume-ng debug mode output
=====================
root@md-trngpoc1:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT#
bash -x bin/flume-ng agent -n agent1 -c conf -f conf/agent1.conf
+ FLUME_AGENT_CLASS=org.apache.flume.node.Application
+ FLUME_AVRO_CLIENT_CLASS=org.apache.flume.client.avro.AvroCLIClient
+
FLUME_CLASSPATH=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf
+
FLUME_JAVA_LIBRARY_PATH=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib
+ JAVA_OPTS=-Xmx20m
+ opt_conf=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf
+ opt_classpath=
+ opt_java_props=
+ opt_dryrun=
+ mode=agent
+ shift
+ case "$mode" in
+ opt_agent=1
+ '[' -n '-n agent1 -c conf -f conf/agent1.conf' ']'
+ arg=-n
+ shift
+ case "$arg" in
+ args=' -n'
+ '[' -n 'agent1 -c conf -f conf/agent1.conf' ']'
+ arg=agent1
+ shift
+ case "$arg" in
+ args=' -n agent1'
+ '[' -n '-c conf -f conf/agent1.conf' ']'
+ arg=-c
+ shift
+ case "$arg" in
+ '[' -n conf ']'
+ opt_conf=conf
+ shift
+ '[' -n '-f conf/agent1.conf' ']'
+ arg=-f
+ shift
+ case "$arg" in
+ args=' -n agent1 -f'
+ '[' -n conf/agent1.conf ']'
+ arg=conf/agent1.conf
+ shift
+ case "$arg" in
+ args=' -n agent1 -f conf/agent1.conf'
+ '[' -n '' ']'
+ [[ -n conf ]]
+ [[ -d conf ]]
++ cd conf
++ pwd
+ opt_conf=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf
+ '[' -z /usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf ']'
+ '[' -f
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh
']'
+ info 'Sourcing environment configuration script
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh'
+ local 'msg=Sourcing environment configuration script
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh'
+ echo 'Info: Sourcing environment configuration script
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh'
Info: Sourcing environment configuration script
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh
+ source
/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh
++ JAVA_HOME=/usr/lib/jvm/java-6-sun-1.6.0.26/jre
++ JAVA_OPTS='-Xms100m -Xmx200m'
++
FLUME_CLASSPATH=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf
++ HADOOP_HOME=/usr/local/hadoop
+ '[' -n '' ']'
+ '[' -n '' ']'
+ '[' -z '' ']'
+++ dirname bin/flume-ng
++ cd bin/..
++ pwd
+ FLUME_HOME=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT
+ '[' -n /usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf ']'
+
FLUME_CLASSPATH='/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib/*:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf'
+ '[' -z /usr/lib/jvm/java-6-sun-1.6.0.26/jre ']'
+ add_hadoop_paths
++
PATH=/usr/local/hadoop/bin:/usr/lib/jvm/java-6-sun/bin:/usr/local/flume_dir/apache-maven-3.0.4/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/hadoop-install/hadoop/bin
++ which hadoop
+ local HADOOP_IN_PATH=/usr/local/hadoop/bin/hadoop
+ '[' -f /usr/local/hadoop/bin/hadoop ']'
+ info 'Including Hadoop libraries found via (/usr/local/hadoop/bin/hadoop)
for HDFS access'
+ local 'msg=Including Hadoop libraries found via
(/usr/local/hadoop/bin/hadoop) for HDFS access'
+ echo 'Info: Including Hadoop libraries found via
(/usr/local/hadoop/bin/hadoop) for HDFS access'
Info: Including Hadoop libraries found via (/usr/local/hadoop/bin/hadoop)
for HDFS access
+ local HADOOP_CLASSPATH=/usr/local/hadoop/bin/hadoop
++
HADOOP_CLASSPATH='/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib/*:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf'
++ /usr/local/hadoop/bin/hadoop org.apache.flume.tools.GetJavaProperty
java.library.path
+ local
HADOOP_JAVA_LIBRARY_PATH=java.library.path=/usr/local/hadoop/bin/../lib/native/Linux-i386-32
+ IFS='
'
+ for line in '$HADOOP_JAVA_LIBRARY_PATH'
+ [[ java.library.path=/usr/local/hadoop/bin/../lib/native/Linux-i386-32 =~
^java\.library\.path=(.*)$ ]]
+ HADOOP_JAVA_LIBRARY_PATH=/usr/local/hadoop/bin/../lib/native/Linux-i386-32
+ break
+ unset IFS
+ '[' -n /usr/local/hadoop/bin/../lib/native/Linux-i386-32 ']'
+
FLUME_JAVA_LIBRARY_PATH=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib:/usr/local/hadoop/bin/../lib/native/Linux-i386-32
++ /usr/local/hadoop/bin/hadoop classpath
+ HADOOP_CLASSPATH=
++ sed -e 's/:/ /g'
+ local ELEMENTS=
+ local ELEMENT
+ add_HBASE_paths
++
PATH=/bin:/usr/lib/jvm/java-6-sun/bin:/usr/local/flume_dir/apache-maven-3.0.4/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/hadoop-install/hadoop/bin
++ which hbase
+ local HBASE_IN_PATH=
+ '[' -f '' ']'
+ '[' -n /usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf ']'
+
FLUME_CLASSPATH='/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib/*:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf'
+ EXEC=exec
+ '[' -n '' ']'
+ '[' -n 1 ']'
+ run_flume org.apache.flume.node.Application -n agent1 -f conf/agent1.conf
+ local FLUME_APPLICATION_CLASS
+ '[' 5 -gt 0 ']'
+ FLUME_APPLICATION_CLASS=org.apache.flume.node.Application
+ shift
+ set -x
+ exec /usr/lib/jvm/java-6-sun-1.6.0.26/jre/bin/java -Xms100m -Xmx200m -cp
'/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib/*:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf'
-Djava.library.path=/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib:/usr/local/hadoop/bin/../lib/native/Linux-i386-32
org.apache.flume.node.Application -n agent1 -f conf/agent1.conf


flume.log
==================
root@md-trngpoc1:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT#
more flume.log
2012-07-04 14:58:30,690 INFO lifecycle.LifecycleSupervisor: Starting
lifecycle supervisor 1
2012-07-04 14:58:30,691 INFO node.FlumeNode: Flume node starting - agent1
2012-07-04 14:58:30,694 INFO nodemanager.DefaultLogicalNodeManager: Node
manager starting
2012-07-04 14:58:30,694 INFO lifecycle.LifecycleSupervisor: Starting
lifecycle supervisor 10
2012-07-04 14:58:30,694 INFO
properties.PropertiesFileConfigurationProvider: Configuration provider
starting
2012-07-04 14:58:30,696 INFO
properties.PropertiesFileConfigurationProvider: Reloading configuration
file:conf/agent1.conf
2012-07-04 14:58:30,702 INFO conf.FlumeConfiguration: Processing:HDFS
2012-07-04 14:58:30,703 INFO conf.FlumeConfiguration: Processing:HDFS
2012-07-04 14:58:30,703 INFO conf.FlumeConfiguration: Processing:HDFS
2012-07-04 14:58:30,703 INFO conf.FlumeConfiguration: Processing:HDFS
2012-07-04 14:58:30,703 INFO conf.FlumeConfiguration: Added sinks: HDFS
Agent: agent1
2012-07-04 14:58:30,718 INFO conf.FlumeConfiguration: Post-validation flume
configuration contains configuration  for agents: [agent1]
2012-07-04 14:58:30,718 INFO
properties.PropertiesFileConfigurationProvider: Creating channels
2012-07-04 14:58:30,722 INFO
properties.PropertiesFileConfigurationProvider: created channel
MemoryChannel-2
2012-07-04 14:58:30,730 INFO sink.DefaultSinkFactory: Creating instance of
sink HDFS typehdfs
Kindly let me know where i am doing wrong?

Thanks,
Vijay

On Tue, Jul 3, 2012 at 10:14 PM, Mike Percy <mpercy@cloudera.com> wrote:

> On Mon, Jul 2, 2012 at 12:11 AM, vijay k <k.vijay52@gmail.com> wrote:
>
>>
>>
>> root@md-trngpoc1:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT#
>> bin/flume-ng agent -n agent1 -c conf -f conf/agent1.conf
>> Info: Sourcing environment configuration script
>> /usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf/flume-env.sh
>> Info: Including Hadoop libraries found via (/usr/local/hadoop/bin/hadoop)
>> for HDFS access
>> + exec /usr/lib/jvm/java-6-sun-1.6.0.26/jre/bin/java -Xms100m -Xmx200m
>> -cp
>> '/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/lib/*:/usr/local/flume_dir/flume/flume-1.2.0-incubating-SNAPSHOT/conf'
>> -Djava.library.path=:/usr/local/hadoop/bin/../lib/native/Linux-i386-32
>> org.apache.flume.node.Application -n agent1 -f conf/agent1.conf
>>
>
> This is really strange. Your java.library.path is set to
> /usr/local/hadoop/lib/native/Linux-i386-32 but for some reason the output
> of "hadoop classpath" has not made it into your classpath. I've never seen
> that combination before - usually both are there or both are missing.
> That's because the same hadoop binary is used for both cases. Are you on a
> 32-bit OS? I don't know if anyone has ever tested Flume on a 32-bit OS.
>
> If you can post the output of the following 6 commands, it might give us
> more to go on:
>
> which hadoop
> echo $HADOOP_HOME
> hadoop classpath
> which java
> echo $JAVA_HOME
> java -version
>
> If you are fluent in shell scripting, you can also trace the bin/flume-ng
> script and see where it's going wrong. Or just invoke it as:
>
>   bash -x bin/flume-ng agent -n agent1 -c conf -f conf/agent1.conf
>
> And post the output of that as well (it will be quite long).
>
> Regards,
> Mike
>
>

Mime
View raw message