cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "n0rad (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CASSANDRA-11594) Too many open files on directories
Date Tue, 16 Aug 2016 10:48:20 GMT

    [ https://issues.apache.org/jira/browse/CASSANDRA-11594?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15422580#comment-15422580
] 

n0rad commented on CASSANDRA-11594:
-----------------------------------

Sorry It's actually 2 keyspaces : 

{code}
CREATE KEYSPACE email_logs_query WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1':
'1', 'DC2': '3'}  AND durable_writes = true;

CREATE TYPE email_logs_query.click_user_type (
    created_at timeuuid,
    link_category text
);

CREATE TABLE email_logs_query.emails (
    email_id text,
    user_id blob,
    clicks list<frozen<click_user_type>>,
    created_at timeuuid,
    email text,
    lang text,
    opens list<timeuuid>,
    sender text,
    site_id int,
    type text,
    PRIMARY KEY (email_id, user_id)
) WITH CLUSTERING ORDER BY (user_id ASC)
    AND bloom_filter_fp_chance = 0.1
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 864000
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';

CREATE MATERIALIZED VIEW email_logs_query.emails_by_user_id AS
    SELECT *
    FROM email_logs_query.emails
    WHERE user_id IS NOT NULL AND created_at IS NOT NULL AND email_id IS NOT NULL
    PRIMARY KEY (user_id, created_at, email_id)
    WITH CLUSTERING ORDER BY (created_at DESC, email_id ASC)
    AND bloom_filter_fp_chance = 0.01
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 864000
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';
{code}

{code}
CREATE KEYSPACE email_logs_command WITH replication = {'class': 'NetworkTopologyStrategy',
'DC1': '1', 'DC2': '3'}  AND durable_writes = true;

CREATE TABLE email_logs_command.events (
    bucket bigint,
    date timeuuid,
    event_type text,
    version int,
    event text,
    PRIMARY KEY (bucket, date, event_type, version)
) WITH CLUSTERING ORDER BY (date ASC, event_type ASC, version ASC)
    AND bloom_filter_fp_chance = 0.01
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 864000
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';
{code}


> Too many open files on directories
> ----------------------------------
>
>                 Key: CASSANDRA-11594
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-11594
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Core
>            Reporter: n0rad
>            Priority: Critical
>         Attachments: openfiles.zip, screenshot.png
>
>
> I have a 6 nodes cluster in prod in 3 racks.
> each node :
> - 4Gb commitlogs on 343 files
> - 275Gb data on 504 files 
> On saturday, 1 node in each rack crash with with too many open files (seems to be the
similar node in each rack).
> {code}
> lsof -n -p $PID give me 66899 out of 65826 max
> {code}
> it contains 64527 open directories (2371 uniq)
> a part of the list :
> {code}
> java    19076 root 2140r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2141r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2142r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2143r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2144r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2145r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2146r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2147r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2148r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2149r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2150r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2151r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2152r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2153r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2154r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java    19076 root 2155r      DIR   8,17      143360 4386718705 /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> {code}
> The 3 others nodes crashes 4 hours later



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message