cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Rajesh Radhakrishnan (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CASSANDRA-12735) org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out
Date Fri, 30 Sep 2016 14:59:20 GMT

    [ https://issues.apache.org/jira/browse/CASSANDRA-12735?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15536214#comment-15536214
] 

Rajesh Radhakrishnan commented on CASSANDRA-12735:
--------------------------------------------------

MY  Cassandra settings: 
Node configuration:[allocate_tokens_for_keyspace=null;
 authenticator=PasswordAuthenticator;
 authorizer=CassandraAuthorizer;
 auto_bootstrap=true;
 auto_snapshot=true;
 batch_size_fail_threshold_in_kb=100;
 batch_size_warn_threshold_in_kb=10;
 batchlog_replay_throttle_in_kb=1024;
 broadcast_address=null;
 broadcast_rpc_address=null;
 buffer_pool_use_heap_if_exhausted=true;
 cas_contention_timeout_in_ms=5000;
 client_encryption_options=<REDACTED>;
 cluster_name=VCF Stress Cluster;
 column_index_cache_size_in_kb=2;
 column_index_size_in_kb=64;
 commit_failure_policy=stop;
 commitlog_compression=null;
 commitlog_directory=null;
 commitlog_max_compression_buffers_in_pool=3;
 commitlog_periodic_queue_size=-1;
 commitlog_segment_size_in_mb=32;
 commitlog_sync=periodic;
 commitlog_sync_batch_window_in_ms=null;
 commitlog_sync_period_in_ms=10000;
 commitlog_total_space_in_mb=null;
 compaction_large_partition_warning_threshold_mb=150;
 compaction_throughput_mb_per_sec=20;
 concurrent_compactors=null;
 concurrent_counter_writes=32;
 concurrent_materialized_view_writes=32;
 concurrent_reads=32;
 concurrent_replicates=null;
 concurrent_writes=32;
 counter_cache_keys_to_save=2147483647;
 counter_cache_save_period=7200;
 counter_cache_size_in_mb=null;
 counter_write_request_timeout_in_ms=5000;
 credentials_cache_max_entries=1000;
 credentials_update_interval_in_ms=-1;
 credentials_validity_in_ms=2000;
 cross_node_timeout=false;
 data_file_directories=[Ljava.lang.String;
@36fc695d;
 disk_access_mode=auto;
 disk_failure_policy=stop;
 disk_optimization_estimate_percentile=0.95;
 disk_optimization_page_cross_chance=0.1;
 disk_optimization_strategy=spinning;
 dynamic_snitch=true;
 dynamic_snitch_badness_threshold=0.1;
 dynamic_snitch_reset_interval_in_ms=600000;
 dynamic_snitch_update_interval_in_ms=100;
 enable_scripted_user_defined_functions=false;
 enable_user_defined_functions=false;
 enable_user_defined_functions_threads=true;
 encryption_options=null;
 endpoint_snitch=SimpleSnitch;
 file_cache_size_in_mb=null;
 gc_warn_threshold_in_ms=1000;
 hinted_handoff_disabled_datacenters=[];
 hinted_handoff_enabled=true;
 hinted_handoff_throttle_in_kb=1024;
 hints_compression=null;
 hints_directory=null;
 hints_flush_period_in_ms=10000;
 incremental_backups=false;
 index_interval=null;
 index_summary_capacity_in_mb=null;
 index_summary_resize_interval_in_minutes=60;
 initial_token=null;
 inter_dc_stream_throughput_outbound_megabits_per_sec=200;
 inter_dc_tcp_nodelay=false;
 internode_authenticator=null;
 internode_compression=dc;
 internode_recv_buff_size_in_bytes=null;
 internode_send_buff_size_in_bytes=null;
 key_cache_keys_to_save=2147483647;
 key_cache_save_period=14400;
 key_cache_size_in_mb=null;
 listen_address=IP.35;
 listen_interface=null;
 listen_interface_prefer_ipv6=false;
 listen_on_broadcast_address=false;
 max_hint_window_in_ms=10800000;
 max_hints_delivery_threads=2;
 max_hints_file_size_in_mb=128;
 max_mutation_size_in_kb=null;
 max_streaming_retries=3;
 max_value_size_in_mb=256;
 memtable_allocation_type=offheap_objects;
 memtable_cleanup_threshold=null;
 memtable_flush_writers=1;
 memtable_heap_space_in_mb=null;
 memtable_offheap_space_in_mb=null;
 native_transport_max_concurrent_connections=-1;
 native_transport_max_concurrent_connections_per_ip=-1;
 native_transport_max_frame_size_in_mb=256;
 native_transport_max_threads=128;
 native_transport_port=9042;
 native_transport_port_ssl=null;
 num_tokens=256;
 otc_coalescing_strategy=TIMEHORIZON;
 otc_coalescing_window_us=200;
 partitioner=org.apache.cassandra.dht.Murmur3Partitioner;
 permissions_cache_max_entries=1000;
 permissions_update_interval_in_ms=-1;
 permissions_validity_in_ms=10000;
 phi_convict_threshold=8.0;
 prepared_statements_cache_size_mb=null;
 range_request_timeout_in_ms=11000;
 read_request_timeout_in_ms=6000;
 request_scheduler=org.apache.cassandra.scheduler.NoScheduler;
 request_scheduler_id=null;
 request_scheduler_options=null;
 request_timeout_in_ms=20000;
 role_manager=CassandraRoleManager;
 roles_cache_max_entries=1000;
 roles_update_interval_in_ms=-1;
 roles_validity_in_ms=5000;
 row_cache_class_name=org.apache.cassandra.cache.OHCProvider;
 row_cache_keys_to_save=2147483647;
 row_cache_save_period=0;
 row_cache_size_in_mb=0;
 rpc_address=IP.35;
 rpc_interface=null;
 rpc_interface_prefer_ipv6=false;
 rpc_keepalive=true;
 rpc_listen_backlog=50;
 rpc_max_threads=2147483647;
 rpc_min_threads=16;
 rpc_port=9160;
 rpc_recv_buff_size_in_bytes=null;
 rpc_send_buff_size_in_bytes=null;
 rpc_server_type=sync;
 saved_caches_directory=null;
 seed_provider=org.apache.cassandra.locator.SimpleSeedProvider{seeds=IP.251};
 server_encryption_options=<REDACTED>;
 snapshot_before_compaction=false;
 ssl_storage_port=7001;
 sstable_preemptive_open_interval_in_mb=50;
 start_native_transport=true;
 start_rpc=false;
 storage_port=7000;
 stream_throughput_outbound_megabits_per_sec=250;
 streaming_socket_timeout_in_ms=86400000;
 thrift_framed_transport_size_in_mb=15;
 thrift_max_message_length_in_mb=16;
 thrift_prepared_statements_cache_size_mb=null;
 tombstone_failure_threshold=100000;
 tombstone_warn_threshold=1000;
 tracetype_query_ttl=86400;
 tracetype_repair_ttl=604800;
 transparent_data_encryption_options=org.apache.cassandra.config.TransparentDataEncryptionOptions@28701274;
 trickle_fsync=false;
 trickle_fsync_interval_in_kb=10240;
 truncate_request_timeout_in_ms=60000;
 unlogged_batch_across_partitions_warn_threshold=10;
 user_defined_function_fail_timeout=1500;
 user_defined_function_warn_timeout=500;
 user_function_timeout_policy=die;
 windows_timer_interval=1;
 write_request_timeout_in_ms=10000]


> org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out
> -------------------------------------------------------------------------
>
>                 Key: CASSANDRA-12735
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-12735
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Configuration, Core
>         Environment: Python 2.7.11, Datastax Cassandra 3.7.0  
>            Reporter: Rajesh Radhakrishnan
>             Fix For: 3.7
>
>
> We got a cluster of two nodes running Cassandra.3.7.0 and using client running Python
2.7.11 injecting data from maybe 100 or so jobs. 
> ------------------------------------------------------------------------------------------
> Cache setting can be seen from system.log:
> INFO  [main] 2016-09-30 15:12:50,002 AuthCache.java:172 - (Re)initializing CredentialsCache
(validity period/update interval/max entries) (2000/2000/1000)
> INFO  [SharedPool-Worker-1] 2016-09-30 15:15:09,561 AuthCache.java:172 - (Re)initializing
PermissionsCache (validity period/update interval/max entries) (10000/10000/1000)
> INFO  [SharedPool-Worker-1] 2016-09-30 15:15:24,319 AuthCache.java:172 - (Re)initializing
RolesCache (validity period/update interval/max entries) (5000/5000/1000)
> ===========================================================
> But I am getting the following exception :
> ERROR [SharedPool-Worker-90] 2016-09-30 15:17:20,883 ErrorMessage.java:338 - Unexpected
exception during request
> com.google.common.util.concurrent.UncheckedExecutionException: com.google.common.util.concurrent.UncheckedExecutionException:
java.lang.RuntimeException: org.apache.cassandra.exceptions.ReadTimeoutException: Operation
timed out - received only 0 responses.
> 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2203) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache.get(LocalCache.java:3937) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3941) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4824) ~[guava-18.0.jar:na]
> 	at org.apache.cassandra.auth.AuthCache.get(AuthCache.java:108) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:45)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.AuthenticatedUser.getPermissions(AuthenticatedUser.java:104)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ClientState.authorize(ClientState.java:375) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ClientState.checkPermissionOnResourceChain(ClientState.java:308)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:285)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:272) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:256)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.statements.ModificationStatement.checkAccess(ModificationStatement.java:211)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.statements.BatchStatement.checkAccess(BatchStatement.java:137)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.QueryProcessor.processBatch(QueryProcessor.java:502) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.QueryProcessor.processBatch(QueryProcessor.java:495) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.transport.messages.BatchMessage.execute(BatchMessage.java:217)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507)
[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401)
[apache-cassandra-3.7.0.jar:3.7.0]
> 	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
[netty-all-4.0.36.Final.jar:4.0.36.Final]
> 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:292)
[netty-all-4.0.36.Final.jar:4.0.36.Final]
> 	at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:32)
[netty-all-4.0.36.Final.jar:4.0.36.Final]
> 	at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:283)
[netty-all-4.0.36.Final.jar:4.0.36.Final]
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_73]
> 	at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164)
[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [apache-cassandra-3.7.0.jar:3.7.0]
> 	at java.lang.Thread.run(Thread.java:745) [na:1.8.0_73]
> Caused by: com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only
0 responses.
> 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2203) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache.get(LocalCache.java:3937) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3941) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4824) ~[guava-18.0.jar:na]
> 	at org.apache.cassandra.auth.AuthCache.get(AuthCache.java:108) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.RolesCache.getRoles(RolesCache.java:44) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.Roles.hasSuperuserStatus(Roles.java:51) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:71) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:86)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.PermissionsCache.lambda$new$1(PermissionsCache.java:37)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.AuthCache$1.load(AuthCache.java:183) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2319) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2282)
~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2197) ~[guava-18.0.jar:na]
> 	... 26 common frames omitted
> Caused by: java.lang.RuntimeException: org.apache.cassandra.exceptions.ReadTimeoutException:
Operation timed out - received only 0 responses.
> 	at org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:489)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.CassandraRoleManager.getRoles(CassandraRoleManager.java:269)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.RolesCache.lambda$new$195(RolesCache.java:36) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.AuthCache$1.load(AuthCache.java:183) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2319) ~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2282)
~[guava-18.0.jar:na]
> 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2197) ~[guava-18.0.jar:na]
> 	... 40 common frames omitted
> Caused by: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out
- received only 0 responses.
> 	at org.apache.cassandra.service.ReadCallback.awaitResults(ReadCallback.java:132) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.ReadCallback.get(ReadCallback.java:137) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.AbstractReadExecutor.get(AbstractReadExecutor.java:145)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.StorageProxy$SinglePartitionReadLifecycle.awaitResultsAndRetryOnDigestMismatch(StorageProxy.java:1714)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1663) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1604) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1523) ~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.db.SinglePartitionReadCommand$Group.execute(SinglePartitionReadCommand.java:979)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:273)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:234)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.CassandraRoleManager.getRoleFromTable(CassandraRoleManager.java:497)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	at org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:485)
~[apache-cassandra-3.7.0.jar:3.7.0]
> 	... 47 common frames omitted



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message