cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Jacek Furmankiewicz (JIRA)" <j...@apache.org>
Subject [jira] [Comment Edited] (CASSANDRA-4687) Exception: DecoratedKey(xxx, yyy) != DecoratedKey(zzz, kkk)
Date Fri, 06 Dec 2013 12:14:09 GMT

    [ https://issues.apache.org/jira/browse/CASSANDRA-4687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13841216#comment-13841216
] 

Jacek Furmankiewicz edited comment on CASSANDRA-4687 at 12/6/13 12:12 PM:
--------------------------------------------------------------------------

Here's a sample of our logs from one of the failed nodes before it stopped responding altogether

{quote}

ERROR [ReadStage:658] 2013-12-05 19:05:13,414 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:658,5,main]
java.lang.AssertionError: DecoratedKey(-1, ) != DecoratedKey(84818348637167105300405141302451086940,
323031342d30332d31325f4c585f33363130) in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:605] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:605,5,main]
java.lang.AssertionError: DecoratedKey(165404967731480145169372669349926870701, 460b000f0000000333323102001000020011000b001400000001300b001500000001310b0016000000024e4f02001a000b001d000000014c0b001e000000024c4802002200020024010b0025000000024c48080026000004a70e00280c000000060b00010000000159000b00010000000158000b00010000000156000b0001000000014c000b00010000000148000b00010000000143000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a0000000353534d06002b00010b0030000000014a0b003100000002325902003201000012323031332d30392d31305f4c585f33393238000000000000029c0000001000000003000000010200000004080000000000007fffffff800000000000000000000001001a42534c5f4d55435f4c585f333932385f323031332d30392d31300200278d0052c833e20004eccbda7b4acc000002430b00010000001a42534c5f4d55435f4c585f333932385f323031332d30392d31300b00020000000342534c0b0003000000034d55430b0004000000024c5808000500000f580b0006000000000b0007000000064c58333932380c000806000100710600020008060003000a06000400100600050019000c000906000100710600020008060003000a060004000e0600050019000c000a06000100710600020008060003000a06000400110600050019000c000b06000100710600020008060003000a060004000f06000500190006000c000106000d000006000e003c0b000f0000000343523702001000020011000b001400000001300b001500000001320b0016000000024e4f02001a000e001c0b00000001000000014a0b001d000000015a0b001e00000002434c0b001f000000132f4c4820434954594c494e4520464f52204c4802002200020024010b0025000000024c480800260000095d0e00280c000000010b0001000000015908000200000046000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003200000012323031342d30362d32345f4c585f3239303900000000000002820000001000000003000000010000400102000000000000007fffffff800000000000000000000001001a4c55475f5a52485f4c585f323930395f323031342d30362d32340201308a8053d1324c0004eccbe8663820000002290b00010000001a4c55475f5a52485f4c585f323930395f323031342d30362d32340b0002000000034c55470b0003000000035a52480b0004000000024c5808000500000b5d0b0006000000000b0007000000064c58323930390c0008060001007206000200050600030018060004000f0600050014000c0009060001007206000200050600030018060004000d0600050014000c000a0600010072060002000506000300180600040010060005000a000c000b060001007206000200050600030018060004000e060005000a0006000c000006000d003206000e00320b000f0000000353323002001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d00000001530b001e00000002463702002200020024000e00280c000000020b0001000000015908000200000018000b000100000001430800020000001a000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30362d31315f4c585f33363932000000000000029a000000100000000300000001000000000c010000000000007fffffff800000000000000000000001001a4756415f4455535f4c585f333639325f323031342d30362d313102011f670053c00eef0004eccbea802afa000002410b00010000001a4756415f4455535f4c585f333639325f323031342d30362d31310b0002000000034756410b0003000000034455530b0004000000024c5808000500000e6c0b0006000000000b0007000000064c58333639320c000806000100720600020005060003000b060004000c060005000a000c000906000100720600020005060003000b060004000a060005000a000c000a06000100720600020005060003000b0600
04000d0600050014000c000b06000100720600020005060003000b060004000b06000500140006000c000106000d000a06000e00460b000f0000000343523902001000020011000b0014000000014d0b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d000000015a0b001e0000000245570b001f000000112f4555524f57494e475320464f52204c4802002200020024010b0025000000024c4808002600000c590e00280c000000010b0001000000015908000200000056000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b003100000002325902003201000011323031342d30342d30345f4c585f373730000000000000026d0000001000000003000000010000400102000000000000007fffffff80000000000000000000000100195a52485f4252555f4c585f3737305f323031342d30342d30340200c5c100536668940004eccbe50d2210000002150b0001000000195a52485f4252555f4c585f3737305f323031342d30342d30340b0002000000035a52480b0003000000034252550b0004000000024c58080005000003020b0006000000000b0007000000054c583737300c00080600010072060002000306000300040600040015000c00090600010072060002000306000300040600040013000c000a0600010072060002000306000300040600040016060005000a000c000b0600010072060002000306000300040600040014060005000a0006000c000106000d000a06000e00460b000f0000000333323002001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001e000000024c5802002200020024000e00280c000000020b000100000001590800020000008a000b0001000000014308000200000014000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30332d30385f4c585f3734323200000000000002800000001000000003000000010040000000100001000000007fffffff800000000000000000000001001a5a52485f5a44485f4c585f373432325f323031342d30332d30380200a228805342d0a30004eccbed971f58000002270b00010000001a5a52485f5a44485f4c585f373432325f323031342d30332d30380b0002000000035a52480b0003000000035a44480b0004000000024c5808000500001cfe0b0006000000000b0007000000064c58373432320c000806000100720600020002060003000806000400120600050004000c000906000100720600020002060003000806000400110600050004000c000a06000100720600020002060003000806000400130600050017000c000b060001007206000200020600030008060004001206000500170006000c000106000d001306000e004f0b000f0000000354524e02001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d00000001580b001f0000000f2f5357495353205241494c574159530f00210c000000010b0001000000014f02000201020003010002002200020024000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000)
!= DecoratedKey(129354628462498787721863245191926267372, 323031342d30332d31325f4c585f36333032)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:621] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:621,5,main]
java.lang.AssertionError: DecoratedKey(20571577737053894014670177383318407961, 0b00000001000000014a0b001d00000001580b001f000000132f4c58)
!= DecoratedKey(11921509772547996733758649988781912542, 323031342d30332d31325f4c585f33363034)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:629] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:629,5,main]
java.lang.AssertionError: DecoratedKey(28342549397252068113495642112118893838, 06000e00370b000f0000000341523102001000020011000b001400000001300b001500000001320b0016000000024e4f02001a000e001c)
!= DecoratedKey(71091626319610127064414775326812359172, 323031342d30332d31325f4c585f31303734)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:655] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:655,5,main]
java.lang.AssertionError: DecoratedKey(141181816010164222522303566125801250182, 0c000000120b0001000000015a000b00010000000159000b00010000000157000b0001000000015600)
!= DecoratedKey(51781751252394853664171685803569647589, 323031342d30332d31325f4c585f33363032)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:624] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:624,5,main]
java.lang.AssertionError: DecoratedKey(117331906676074686904286748592775815144, 00006c000b0001000000014308000200000044000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30382d32355f4c585f33353530000000000000029b0000001000000003000000010000000000001024000000007fffffff800000000000000000000001001a5a52485f5649455f4c585f333535305f323031342d30382d323502018247805422ef5d0004eccbe970e990000002420b00010000001a5a52485f5649455f4c585f333535305f323031342d30382d32350b0002000000035a52480b0003000000035649450b0004000000024c5808000500000dde0b0006000000000b0007000000064c58333535300c00080600010072060002000706000300190600040007060005002d000c00090600010072060002000706000300190600040005060005002d000c000a0600010072060002000706000300190600)
!= DecoratedKey(151509992540710526804531373576561300989, 323031342d30332d31325f4c585f31303638)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
{quote}


was (Author: jfurmankiewicz):
Here's a sample of our logs from one of the failed logs before it stopped responding altogether

{quote}

ERROR [ReadStage:658] 2013-12-05 19:05:13,414 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:658,5,main]
java.lang.AssertionError: DecoratedKey(-1, ) != DecoratedKey(84818348637167105300405141302451086940,
323031342d30332d31325f4c585f33363130) in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:605] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:605,5,main]
java.lang.AssertionError: DecoratedKey(165404967731480145169372669349926870701, 460b000f0000000333323102001000020011000b001400000001300b001500000001310b0016000000024e4f02001a000b001d000000014c0b001e000000024c4802002200020024010b0025000000024c48080026000004a70e00280c000000060b00010000000159000b00010000000158000b00010000000156000b0001000000014c000b00010000000148000b00010000000143000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a0000000353534d06002b00010b0030000000014a0b003100000002325902003201000012323031332d30392d31305f4c585f33393238000000000000029c0000001000000003000000010200000004080000000000007fffffff800000000000000000000001001a42534c5f4d55435f4c585f333932385f323031332d30392d31300200278d0052c833e20004eccbda7b4acc000002430b00010000001a42534c5f4d55435f4c585f333932385f323031332d30392d31300b00020000000342534c0b0003000000034d55430b0004000000024c5808000500000f580b0006000000000b0007000000064c58333932380c000806000100710600020008060003000a06000400100600050019000c000906000100710600020008060003000a060004000e0600050019000c000a06000100710600020008060003000a06000400110600050019000c000b06000100710600020008060003000a060004000f06000500190006000c000106000d000006000e003c0b000f0000000343523702001000020011000b001400000001300b001500000001320b0016000000024e4f02001a000e001c0b00000001000000014a0b001d000000015a0b001e00000002434c0b001f000000132f4c4820434954594c494e4520464f52204c4802002200020024010b0025000000024c480800260000095d0e00280c000000010b0001000000015908000200000046000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003200000012323031342d30362d32345f4c585f3239303900000000000002820000001000000003000000010000400102000000000000007fffffff800000000000000000000001001a4c55475f5a52485f4c585f323930395f323031342d30362d32340201308a8053d1324c0004eccbe8663820000002290b00010000001a4c55475f5a52485f4c585f323930395f323031342d30362d32340b0002000000034c55470b0003000000035a52480b0004000000024c5808000500000b5d0b0006000000000b0007000000064c58323930390c0008060001007206000200050600030018060004000f0600050014000c0009060001007206000200050600030018060004000d0600050014000c000a0600010072060002000506000300180600040010060005000a000c000b060001007206000200050600030018060004000e060005000a0006000c000006000d003206000e00320b000f0000000353323002001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d00000001530b001e00000002463702002200020024000e00280c000000020b0001000000015908000200000018000b000100000001430800020000001a000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30362d31315f4c585f33363932000000000000029a000000100000000300000001000000000c010000000000007fffffff800000000000000000000001001a4756415f4455535f4c585f333639325f323031342d30362d313102011f670053c00eef0004eccbea802afa000002410b00010000001a4756415f4455535f4c585f333639325f323031342d30362d31310b0002000000034756410b0003000000034455530b0004000000024c5808000500000e6c0b0006000000000b0007000000064c58333639320c000806000100720600020005060003000b060004000c060005000a000c000906000100720600020005060003000b060004000a060005000a000c000a06000100720600020005060003000b0600
04000d0600050014000c000b06000100720600020005060003000b060004000b06000500140006000c000106000d000a06000e00460b000f0000000343523902001000020011000b0014000000014d0b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d000000015a0b001e0000000245570b001f000000112f4555524f57494e475320464f52204c4802002200020024010b0025000000024c4808002600000c590e00280c000000010b0001000000015908000200000056000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b003100000002325902003201000011323031342d30342d30345f4c585f373730000000000000026d0000001000000003000000010000400102000000000000007fffffff80000000000000000000000100195a52485f4252555f4c585f3737305f323031342d30342d30340200c5c100536668940004eccbe50d2210000002150b0001000000195a52485f4252555f4c585f3737305f323031342d30342d30340b0002000000035a52480b0003000000034252550b0004000000024c58080005000003020b0006000000000b0007000000054c583737300c00080600010072060002000306000300040600040015000c00090600010072060002000306000300040600040013000c000a0600010072060002000306000300040600040016060005000a000c000b0600010072060002000306000300040600040014060005000a0006000c000106000d000a06000e00460b000f0000000333323002001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001e000000024c5802002200020024000e00280c000000020b000100000001590800020000008a000b0001000000014308000200000014000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30332d30385f4c585f3734323200000000000002800000001000000003000000010040000000100001000000007fffffff800000000000000000000001001a5a52485f5a44485f4c585f373432325f323031342d30332d30380200a228805342d0a30004eccbed971f58000002270b00010000001a5a52485f5a44485f4c585f373432325f323031342d30332d30380b0002000000035a52480b0003000000035a44480b0004000000024c5808000500001cfe0b0006000000000b0007000000064c58373432320c000806000100720600020002060003000806000400120600050004000c000906000100720600020002060003000806000400110600050004000c000a06000100720600020002060003000806000400130600050017000c000b060001007206000200020600030008060004001206000500170006000c000106000d001306000e004f0b000f0000000354524e02001000020011000b001400000001300b001500000001300b0016000000024e4f02001a000e001c0b00000001000000014a0b001d00000001580b001f0000000f2f5357495353205241494c574159530f00210c000000010b0001000000014f02000201020003010002002200020024000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000)
!= DecoratedKey(129354628462498787721863245191926267372, 323031342d30332d31325f4c585f36333032)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:621] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:621,5,main]
java.lang.AssertionError: DecoratedKey(20571577737053894014670177383318407961, 0b00000001000000014a0b001d00000001580b001f000000132f4c58)
!= DecoratedKey(11921509772547996733758649988781912542, 323031342d30332d31325f4c585f33363034)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:629] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:629,5,main]
java.lang.AssertionError: DecoratedKey(28342549397252068113495642112118893838, 06000e00370b000f0000000341523102001000020011000b001400000001300b001500000001320b0016000000024e4f02001a000e001c)
!= DecoratedKey(71091626319610127064414775326812359172, 323031342d30332d31325f4c585f31303734)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:655] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:655,5,main]
java.lang.AssertionError: DecoratedKey(141181816010164222522303566125801250182, 0c000000120b0001000000015a000b00010000000159000b00010000000157000b0001000000015600)
!= DecoratedKey(51781751252394853664171685803569647589, 323031342d30332d31325f4c585f33363032)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
ERROR [ReadStage:624] 2013-12-05 19:05:13,413 AbstractCassandraDaemon.java (line 132) Exception
in thread Thread[ReadStage:624,5,main]
java.lang.AssertionError: DecoratedKey(117331906676074686904286748592775815144, 00006c000b0001000000014308000200000044000e00290c000000120b0001000000015a000b00010000000159000b00010000000157000b00010000000156000b00010000000155000b00010000000154000b00010000000142000b00010000000143000b00010000000144000b00010000000145000b00010000000148000b0001000000014a000b0001000000014b000b0001000000014c000b0001000000014d000b00010000000150000b00010000000151000b00010000000153000b002a000000045353494d06002b00010b0030000000014a0b0031000000024c5802003201000012323031342d30382d32355f4c585f33353530000000000000029b0000001000000003000000010000000000001024000000007fffffff800000000000000000000001001a5a52485f5649455f4c585f333535305f323031342d30382d323502018247805422ef5d0004eccbe970e990000002420b00010000001a5a52485f5649455f4c585f333535305f323031342d30382d32350b0002000000035a52480b0003000000035649450b0004000000024c5808000500000dde0b0006000000000b0007000000064c58333535300c00080600010072060002000706000300190600040007060005002d000c00090600010072060002000706000300190600040005060005002d000c000a0600010072060002000706000300190600)
!= DecoratedKey(151509992540710526804531373576561300989, 323031342d30332d31325f4c585f31303638)
in /app/cassandra-int/data/SCHEDULE/LEG/SCHEDULE-LEG-hf-1-Data.db
	at org.apache.cassandra.db.columniterator.SSTableNamesIterator.<init>(SSTableNamesIterator.java:72)
	at org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:61)
	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
	at org.apache.cassandra.db.CollationController.collectTimeOrderedData(CollationController.java:124)
	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1362)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1224)
	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1159)
	at org.apache.cassandra.db.Table.getRow(Table.java:378)
	at org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:58)
	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:822)
	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1271)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:722)
{quote}

> Exception: DecoratedKey(xxx, yyy) != DecoratedKey(zzz, kkk)
> -----------------------------------------------------------
>
>                 Key: CASSANDRA-4687
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-4687
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Core
>         Environment: CentOS 6.3 64-bit, Oracle JRE 1.6.0.33 64-bit, single node cluster
>            Reporter: Leonid Shalupov
>            Priority: Minor
>         Attachments: 4687-debugging.txt
>
>
> Under heavy write load sometimes cassandra fails with assertion error.
> git bisect leads to commit 295aedb278e7a495213241b66bc46d763fd4ce66.
> works fine if global key/row caches disabled in code.
> {quote}
> java.lang.AssertionError: DecoratedKey(xxx, yyy) != DecoratedKey(zzz, kkk) in /var/lib/cassandra/data/...-he-1-Data.db
> 	at org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:60)
> 	at org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:67)
> 	at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79)
> 	at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:256)
> 	at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64)
> 	at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1345)
> 	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207)
> 	at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1142)
> 	at org.apache.cassandra.db.Table.getRow(Table.java:378)
> 	at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:69)
> 	at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:819)
> 	at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1253)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> 	at java.lang.Thread.run(Thread.java:662)
> {quote}



--
This message was sent by Atlassian JIRA
(v6.1#6144)

Mime
View raw message