incubator-cassandra-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hive13 Wong <hiv...@gmail.com>
Subject cassandra crashed
Date Fri, 11 Jun 2010 13:47:59 GMT
One of our cassandra nodes suddenly crashed, then the other 2...
Exceptions found in the system.log are attached below.
Any ideas? Does it mean that we've got some bad data running around in the
cluster?
Many thanks

The exeption on the node that crashed first was like
ERROR [RESPONSE-STAGE:669] 2010-06-11 21:46:16,197 CassandraDaemon.java
(line 78) Fatal exception in thread Thread[RESPONSE-STAGE:669,5,main]
java.lang.RuntimeException: Error handling responses for
Row(key='be71838def3abf957a2dfc68367f6855', cf=ColumnFamily(crawl_url
[6964:false:32@1275838896468,6c6173744d6f6454696d65:false:1@1275838896468
,6c6174657374437261776c54696d65:false:1@1275838896468
,6e65656443616c63496e416e616c797365:false:1@1275838896468
,6e657874437261776c54696d65:false:1@1275838896468
,73697465:false:29@1275838896468
,756e6368616e67656454696d6573:false:1@1275838896468
,75726c:false:48@1275838896468,]))
        at
org.apache.cassandra.service.ConsistencyChecker$DigestResponseHandler.response(ConsistencyChecker.java:136)
        at
org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:35)
        at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.UTFDataFormatException: malformed input around byte 1
        at java.io.DataInputStream.readUTF(DataInputStream.java:639)
        at java.io.DataInputStream.readUTF(DataInputStream.java:547)
        at org.apache.cassandra.db.RowSerializer.deserialize(Row.java:73)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:114)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:90)
        at
org.apache.cassandra.service.ConsistencyChecker$DigestResponseHandler.response(ConsistencyChecker.java:108)
        ... 5 more


Exceptions on the other 2 nodes are like
ERROR [pool-1-thread-6523] 2010-06-11 21:35:19,243 Cassandra.java (line
1323) Internal error processing multiget_slice
java.lang.NegativeArraySizeException
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:107)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:90)
        at
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:369)
        at
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:410)
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:101)
        at
org.apache.cassandra.thrift.CassandraServer.getSlice(CassandraServer.java:184)
        at
org.apache.cassandra.thrift.CassandraServer.multigetSliceInternal(CassandraServer.java:263)
        at
org.apache.cassandra.thrift.CassandraServer.multiget_slice(CassandraServer.java:235)
        at
org.apache.cassandra.thrift.Cassandra$Processor$multiget_slice.process(Cassandra.java:1315)
        at
org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:1125)
        at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)
ERROR [pool-1-thread-6445] 2010-06-11 21:35:19,243 Cassandra.java (line
1323) Internal error processing multiget_slice
java.lang.RuntimeException: java.io.EOFException
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:109)
        at
org.apache.cassandra.thrift.CassandraServer.getSlice(CassandraServer.java:184)
        at
org.apache.cassandra.thrift.CassandraServer.multigetSliceInternal(CassandraServer.java:263)
        at
org.apache.cassandra.thrift.CassandraServer.multiget_slice(CassandraServer.java:235)
        at
org.apache.cassandra.thrift.Cassandra$Processor$multiget_slice.process(Cassandra.java:1315)
        at
org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:1125)
        at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.EOFException
        at java.io.DataInputStream.readBoolean(DataInputStream.java:227)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:109)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:90)
        at
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:369)
        at
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:410)
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:101)
        ... 9 more

and

ERROR [pool-1-thread-131] 2010-06-11 21:55:49,107 Cassandra.java (line 1239)
Internal error processing get_slice
java.lang.RuntimeException: java.io.EOFException
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:109)
        at
org.apache.cassandra.thrift.CassandraServer.getSlice(CassandraServer.java:184)
        at
org.apache.cassandra.thrift.CassandraServer.multigetSliceInternal(CassandraServer.java:263)
        at
org.apache.cassandra.thrift.CassandraServer.get_slice(CassandraServer.java:224)
        at
org.apache.cassandra.thrift.Cassandra$Processor$get_slice.process(Cassandra.java:1231)
        at
org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:1125)
        at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.EOFException
        at java.io.DataInputStream.readBoolean(DataInputStream.java:227)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:109)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:90)
        at
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:369)
        at
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:410)
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:101)
        ... 9 more
ERROR [pool-1-thread-146] 2010-06-11 21:55:49,560 Cassandra.java (line 1323)
Internal error processing multiget_slice
java.lang.RuntimeException: Unable to load comparator class 'Standard'.
 probably this means you have obsolete sstables lying around
        at
org.apache.cassandra.db.ColumnFamilySerializer.readComparator(ColumnFamilySerializer.java:136)
        at
org.apache.cassandra.db.ColumnFamilySerializer.deserialize(ColumnFamilySerializer.java:107)
        at org.apache.cassandra.db.RowSerializer.deserialize(Row.java:73)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:114)
        at
org.apache.cassandra.db.ReadResponseSerializer.deserialize(ReadResponse.java:90)
        at
org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:369)
        at
org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:410)
        at
org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:101)
        at
org.apache.cassandra.thrift.CassandraServer.getSlice(CassandraServer.java:184)
        at
org.apache.cassandra.thrift.CassandraServer.multigetSliceInternal(CassandraServer.java:263)
        at
org.apache.cassandra.thrift.CassandraServer.multiget_slice(CassandraServer.java:235)
        at
org.apache.cassandra.thrift.Cassandra$Processor$multiget_slice.process(Cassandra.java:1315)
        at
org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:1125)
        at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.ClassNotFoundException: Standard
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)

Mime
View raw message