incubator-cassandra-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Teodor Sigaev <teo...@sigaev.ru>
Subject Re: Cassandra restart
Date Wed, 14 Oct 2009 14:25:43 GMT
Sorry, I don't see anything connected to key 00000000000013062900 or message 
2691, debug is turned off on that node because 10.3.2.36 wasn't restarted

I will try to reproduce problem on smaller test cluster.

Fragment of 10.3.2.36 log (time is in sync):
ERROR [pool-1-thread-15051] 2009-10-14 18:03:30,562 StorageProxy.java (line 179) 
error writing key 00000000000176715312
java.util.concurrent.TimeoutException: Operation timed out - received only 0 
responses from  .
     at 
org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88)
     at 
org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164)
     at 
org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
     at 
org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461)
     at 
org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14947] 2009-10-14 18:03:32,593 StorageProxy.java (line 179) 
error writing key 00000000000176715312
java.util.concurrent.TimeoutException: Operation timed out - received only 0 
responses from  .
     at 
org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88)
     at 
org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164)
     at 
org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
     at 
org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461)
     at 
org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14985] 2009-10-14 18:03:34,571 StorageProxy.java (line 179) 
error writing key 00000000000176715312
java.util.concurrent.TimeoutException: Operation timed out - received only 0 
responses from  .
     at 
org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88)
     at 
org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164)
     at 
org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
     at 
org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461)
     at 
org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14493] 2009-10-14 18:03:34,571 Cassandra.java (line 679) 
Internal error processing get_slice
java.lang.RuntimeException: error reading key 00000000000106371503
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265)
     at 
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312)
     at 
org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100)
     at 
org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182)
     at 
org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251)
     at 
org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220)
     at 
org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
     at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261)
     ... 11 more
ERROR [pool-1-thread-14820] 2009-10-14 18:03:35,406 Cassandra.java (line 679) 
Internal error processing get_slice
java.lang.RuntimeException: error reading key 00000000000019586353
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265)
     at 
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312)
     at 
org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100)
     at 
org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182)
     at 
org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251)
     at 
org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220)
     at 
org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
     at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261)
     ... 11 more
ERROR [pool-1-thread-15051] 2009-10-14 18:03:36,602 StorageProxy.java (line 179) 
error writing key 00000000000176715312
java.util.concurrent.TimeoutException: Operation timed out - received only 0 
responses from  .
     at 
org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88)
     at 
org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164)
     at 
org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
     at 
org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461)
     at 
org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14658] 2009-10-14 18:03:37,590 Cassandra.java (line 679) 
Internal error processing get_slice
java.lang.RuntimeException: error reading key 00000000000139616600
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265)
     at 
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312)
     at 
org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100)
     at 
org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182)
     at 
org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251)
     at 
org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220)
     at 
org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671)
     at org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
     at 
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
     at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
     at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
     at 
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261)
     ... 11 more


-- 
Teodor Sigaev                                   E-mail: teodor@sigaev.ru
                                                    WWW: http://www.sigaev.ru/

Mime
View raw message