couchdb-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Alexey Loshkarev <elf2...@gmail.com>
Subject high IO wait and unresponsible views after couchdb restart
Date Wed, 27 Apr 2011 14:22:26 GMT
Hi folks.

I'm experiencing strange problem with CouchDB.
I have some large databases (about ~30-40 GB with  ~2.5M documents in
it). There also (sure) indexes. Index files are 300-700 MB after
compaction.
Sometimes (it's inreproducible), after restarting couchdb, couchdb
view engine hangs for a while (tens of minutes, sometimes). It means,
no view request may be done. Documents are accessible ok, without a
problems, but view calls hangs.
Also, there is high disc activity.
beam.smp IO Wait is 100% with 3-5 MB/s reads (almost no writes).
There are no messages in couchdb.log
There are no processes at status page of futon.
Replications, if i start them, works with no problems.
View index files are not growing, so I can't guess, wich view is affected.
The only problem is unresponsive view engine.
After a while view engine starts reponse as nothing was happens and
database continue works nice.

CouchDB version is 1.0.2
Erlang - 13.2.4
Gentoo Linux

I tried to strace those process, here are pieces of log:
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428933, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274036736)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428935, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274032640)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428937, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274028544)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428939, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274024448)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428941, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274020352)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428943, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274016256)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428945, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274012160)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428947, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274008064)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428949, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274003968)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428951, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
............
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428351, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275220480)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428353, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275216384)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428355, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275212288)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428357, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275208192)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428359, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275204096)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428361, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275200000)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428363, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275195904)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428365, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275191808)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428367, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275187712)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428369, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275183616)         = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428371, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275179520)         = 1





-- 
----------------
Best regards
Alexey Loshkarev
mailto:elf2001@gmail.com

Mime
View raw message