flume-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "基勇" <252637...@qq.com>
Subject Flume sinkgroups sinks share problem
Date Wed, 13 Aug 2014 03:02:47 GMT
Hello,guys
        I think to config flume Failover and Load balancing.But I found that sinks can not
be shared.
        config file - flume-sink.properties‍‍‍
#Name the compents on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1


#Describe the sinkgroups
a1.sinkgroups = g1 g2 g3
a1.sinkgroups.g1.sinks = k1 k2
a1.sinkgroups.g1.processor.type = failover
a1.sinkgroups.g1.processor.priority.k1 = 5
a1.sinkgroups.g1.processor.priority.k2 = 10
a1.sinkgroups.g1.processor.maxpenalty = 10000


a1.sinkgroups.g2.sinks = k1 k3
a1.sinkgroups.g2.processor.type = load_balance
a1.sinkgroups.g2.processor.backoff = true
a1.sinkgroups.g2.processor.selector = round_robin


a1.sinkgroups.g3.sinks = k3 k4
a1.sinkgroups.g3.processor.type = failover
a1.sinkgroups.g3.processor.priority.k3 = 5
a1.sinkgroups.g3.processor.priority.k4 = 10
a1.sinkgroups.g3.processor.maxpenalty = 10000


#Describe/config the source
a1.sources.r1.type = syslogtcp
a1.sources.r1.port = 5140
a1.sources.r1.host = localhost
a1.sources.r1.channels = c1


#Describe the sink
a1.sinks.k1.type = avro
a1.sinks.k1.channel = c1
a1.sinks.k1.hostname = 192.168.220.159
a1.sinks.k1.port = 44411


a1.sinks.k2.type = avro
a1.sinks.k2.channel = c1
a1.sinks.k2.hostname = 192.168.220.159
a1.sinks.k2.port = 44422


a1.sinks.k3.type = avro
a1.sinks.k3.channel = c1
a1.sinks.k3.hostname = 192.168.220.159
a1.sinks.k3.port = 44433


a1.sinks.k4.type = avro
a1.sinks.k4.channel = c1
a1.sinks.k4.hostname = 192.168.220.159
a1.sinks.k4.port = 44444
#Use a channel which butters events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100‍


execute the running command,no running success. 
here is log:

2014-07-06 14:30:01,100 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validGroupSinks(FlumeConfiguration.java:814)]
Agent configuration for 'a1' sinkgroup 'g2' sink not found: 'k3',  sink not added

2014-07-06 14:30:01,103 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validGroupSinks(FlumeConfiguration.java:803)]
Agent configuration for 'a1' sinkgroup 'g1' sink 'k1' in use by another group: 'g2', sink
not added

2014-07-06 14:30:01,104 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validGroupSinks(FlumeConfiguration.java:814)]
Agent configuration for 'a1' sinkgroup 'g1' sink not found: 'k2',  sink not added

2014-07-06 14:30:01,105 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateGroups(FlumeConfiguration.java:764)]
Could not configure sink group g1 due to: No available sinks for sinkgroup: g1. Sinkgroup
will be removed

org.apache.flume.conf.ConfigurationException: No available sinks for sinkgroup: g1. Sinkgroup
will be removed

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateGroups(FlumeConfiguration.java:753)

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.isValid(FlumeConfiguration.java:347)

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.access$000(FlumeConfiguration.java:212)

        at org.apache.flume.conf.FlumeConfiguration.validateConfiguration(FlumeConfiguration.java:126)

        at org.apache.flume.conf.FlumeConfiguration.<init>(FlumeConfiguration.java:108)

        at org.apache.flume.node.PropertiesFileConfigurationProvider.getFlumeConfiguration(PropertiesFileConfigurationProvider.java:193)

        at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:94)

        at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:140)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)

        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

2014-07-06 14:30:01,114 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validGroupSinks(FlumeConfiguration.java:814)]
Agent configuration for 'a1' sinkgroup 'g3' sink not found: 'k3',  sink not added

2014-07-06 14:30:01,115 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validGroupSinks(FlumeConfiguration.java:814)]
Agent configuration for 'a1' sinkgroup 'g3' sink not found: 'k4',  sink not added

2014-07-06 14:30:01,116 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateGroups(FlumeConfiguration.java:764)]
Could not configure sink group g3 due to: No available sinks for sinkgroup: g3. Sinkgroup
will be removed

org.apache.flume.conf.ConfigurationException: No available sinks for sinkgroup: g3. Sinkgroup
will be removed

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateGroups(FlumeConfiguration.java:753)

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.isValid(FlumeConfiguration.java:347)

        at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.access$000(FlumeConfiguration.java:212)

        at org.apache.flume.conf.FlumeConfiguration.validateConfiguration(FlumeConfiguration.java:126)

        at org.apache.flume.conf.FlumeConfiguration.<init>(FlumeConfiguration.java:108)

        at org.apache.flume.node.PropertiesFileConfigurationProvider.getFlumeConfiguration(PropertiesFileConfigurationProvider.java:193)

        at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:94)

        at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:140)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)

        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

2014-07-06 14:30:01,119 (conf-file-poller-0) [INFO - org.apache.flume.conf.FlumeConfiguration.validateConfiguration(FlumeConfiguration.java:140)]
Post-validation flume configuration contains configuration for agents: [a1]

2014-07-06 14:30:01,120 (conf-file-poller-0) [INFO - org.apache.flume.node.AbstractConfigurationProvider.loadChannels(AbstractConfigurationProvider.java:150)]
Creating channels

2014-07-06 14:30:01,129 (conf-file-poller-0) [INFO - org.apache.flume.channel.DefaultChannelFactory.create(DefaultChannelFactory.java:40)]
Creating instance of channel c1 type memory

2014-07-06 14:30:01,134 (conf-file-poller-0) [INFO - org.apache.flume.node.AbstractConfigurationProvider.loadChannels(AbstractConfigurationProvider.java:205)]
Created channel c1

2014-07-06 14:30:01,135 (conf-file-poller-0) [INFO - org.apache.flume.source.DefaultSourceFactory.create(DefaultSourceFactory.java:39)]
Creating instance of source r1, type syslogtcp

2014-07-06 14:30:01,178 (conf-file-poller-0) [INFO - org.apache.flume.sink.DefaultSinkFactory.create(DefaultSinkFactory.java:40)]
Creating instance of sink: k1, type: avro

2014-07-06 14:30:01,182 (conf-file-poller-0) [INFO - org.apache.flume.sink.AbstractRpcSink.configure(AbstractRpcSink.java:183)]
Connection reset is set to 0. Will not reset connection to next hop

2014-07-06 14:30:01,185 (conf-file-poller-0) [ERROR - org.apache.flume.node.AbstractConfigurationProvider.loadSinkGroups(AbstractConfigurationProvider.java:478)]
SinkGroup g2 has been removed due to an error during configuration

java.lang.IllegalStateException: The LoadBalancingSinkProcessor cannot be used for a single
sink. Please configure more than one sinks and try again.

        at com.google.common.base.Preconditions.checkState(Preconditions.java:145)

        at org.apache.flume.sink.LoadBalancingSinkProcessor.configure(LoadBalancingSinkProcessor.java:97)

        at org.apache.flume.conf.Configurables.configure(Configurables.java:41)

        at org.apache.flume.sink.SinkProcessorFactory.getProcessor(SinkProcessorFactory.java:95)

        at org.apache.flume.sink.SinkGroup.configure(SinkGroup.java:65)

        at org.apache.flume.conf.Configurables.configure(Configurables.java:50)

        at org.apache.flume.node.AbstractConfigurationProvider.loadSinkGroups(AbstractConfigurationProvider.java:472)

        at org.apache.flume.node.AbstractConfigurationProvider.loadSinks(AbstractConfigurationProvider.java:437)

        at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:103)

        at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:140)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)

        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

2014-07-06 14:30:01,189 (conf-file-poller-0) [INFO - org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:119)]
Channel c1 connected to [r1, k1]‍
Mime
View raw message