ReeceStevens commented on issue #1093: Hanging or 500 errors during concurrent replication URL: https://github.com/apache/couchdb/issues/1093#issuecomment-359965144 Hi there @wohali , Preface: I only had couchDB 2.0.0 to work with when trying your suggestions, but I believe the underlying behavior is identical to v2.1.1. So I first watched message queues while the problem occurred-- I'm not sure what a relatively large message queue size is for couchDB but the only ones that rose significantly were `file_server_2` and `couch_index_server` ("significantly" meaning on the order of 100). However, this was always a transient increase, and by the time couch started throwing 500 errors they were both back at zero. I never saw `couch_server` increase at all. FWIW, I will post the `_system` node response after couch crashes at the bottom of this response. Next, I ran with the `update_lru_on_read = false` option set in my `local.ini`. This seemed to trigger the error more quickly. Before adding the option, couchDB would take about 24-26 iterations of my parallel replication script before crashing; after, it only took 4. This was consistent across multiple trials. ### `_system` node response with `update_lru_on_read` not set ``` { "uptime": 228, "memory": { "other": 108667722, "atom": 504409, "atom_used": 472138, "processes": 34621392, "processes_used": 34608264, "binary": 813285896, "code": 12084677, "ets": 1842064 }, "run_queue": 0, "ets_table_count": 86, "context_switches": 7444084, "reductions": 1233973042, "garbage_collection_count": 842909, "words_reclaimed": 1573709799, "io_input": 7708805364, "io_output": 6367883539, "os_proc_count": 0, "stale_proc_count": 0, "process_count": 876, "process_limit": 262144, "message_queues": { "couch_file": { "50": 0, "90": 0, "99": 0, "count": 216, "min": 0, "max": 0 }, "couch_db_updater": { "50": 0, "90": 0, "99": 0, "count": 176, "min": 0, "max": 0 }, "error_logger": 0, "couch_auth_cache": 0, "mango_sup": 0, "httpd_sup": 0, "couch_log_sup": 0, "couch_log_server": 0, "erl_prim_loader": 0, "global_group": 0, "standard_error_sup": 0, "erl_epmd": 0, "couch_epi_sup": 0, "couch_epi_functions_gen_couch_index": 0, "couch_epi_functions_gen_chttpd_auth": 0, "couch_epi_functions_gen_couch_db": 0, "couch_epi_functions_gen_global_changes": 0, "kernel_safe_sup": 0, "couch_epi_functions_gen_chttpd_handlers": 0, "couch_epi_functions_gen_chttpd": 0, "auth": 0, "code_server": 0, "rexi_buffer_couchdb@localhost": 0, "rexi_server_couchdb@localhost": 0, "mem3_sync_nodes": 0, "application_controller": 0, "httpc_sup": 0, "mochiweb_clock": 0, "mem3_sync": 0, "config_sup": 0, "mem3_sup": 0, "config": 0, "httpc_profile_sup": 0, "mem3_shards": 0, "global_changes_sup": 0, "inet_gethost_native_sup": 0, "httpc_manager": 0, "global_changes_server": 0, "httpc_handler_sup": 0, "runtime_tools_sup": 0, "mem3_nodes": 0, "net_kernel": 0, "user": 0, "ddoc_cache_sup": 0, "ftp_sup": 0, "rexi_sup": 0, "ddoc_cache_opener": 0, "rexi_server_sup": 0, "rexi_server_mon": 0, "rexi_server": 0, "ddoc_cache_lru": 0, "ioq_sup": 0, "chttpd_auth_cache_lru": 0, "memsup": 0, "mem3_events": 0, "disksup": 0, "ioq": 0, "couch_secondary_services": 0, "cpu_sup": 0, "couch_primary_services": 0, "couch_replicator_sup": 0, "ibrowse_sup": 0, "couch_replicator_manager": 0, "couch_replicator_job_sup": 0, "couch_uuids": 0, "rexi_buffer_mon": 0, "config_event": 0, "rexi_buffer_sup": 0, "ibrowse": 0, "couch_event_sup2": 0, "couch_event_server": 0, "couch_task_status": 0, "couch_event_os_sup": 0, "couch_sup": 0, "release_handler": 0, "couch_server": 0, "os_mon_sup": 0, "overload": 0, "couch_proc_manager": 0, "sasl_sup": 0, "couch_stats_sup": 0, "couch_stats_process_tracker": 0, "couch_plugin": 0, "couch_replication": 0, "alarm_handler": 0, "couch_os_daemons": 0, "tftp_sup": 0, "sasl_safe_sup": 0, "couch_stats_aggregator": 0, "timer_server": 0, "couch_httpd_vhost": 0, "folsom_sup": 0, "couch_index_sup": 0, "folsom_sample_slide_sup": 0, "couch_index_server": 0, "couch_httpd": 0, "rex": 0, "standard_error": 0, "init": 0, "net_sup": 0, "tls_connection_sup": 0, "chttpd_sup": 0, "kernel_sup": 0, "couch_external_manager": 0, "global_name_server": 0, "folsom_metrics_histogram_ets": 0, "file_server_2": 0, "couch_drv": 0, "ssl_sup": 0, "folsom_meter_timer_server": 0, "chttpd_auth_cache": 0, "setup_sup": 0, "inet_gethost_native": 0, "inet_db": 0, "inets_sup": 0, "ssl_manager": 0, "ssl_listen_tracker_sup": 0, "couch_compaction_daemon": 0, "chttpd": 0 }, "internal_replication_jobs": 0, "distribution": {} } ``` ### `_system` node response with `update_lru_on_read = false` ``` { "uptime": 357, "memory": { "other": 107225738, "atom": 504409, "atom_used": 472138, "processes": 26205048, "processes_used": 26174352, "binary": 572270512, "code": 12084677, "ets": 1700232 }, "run_queue": 0, "ets_table_count": 86, "context_switches": 5537983, "reductions": 867477179, "garbage_collection_count": 787554, "words_reclaimed": 1465399318, "io_input": 5019941144, "io_output": 2954750043, "os_proc_count": 0, "stale_proc_count": 0, "process_count": 736, "process_limit": 262144, "message_queues": { "couch_file": { "50": 0, "90": 0, "99": 0, "count": 168, "min": 0, "max": 0 }, "couch_db_updater": { "50": 0, "90": 0, "99": 0, "count": 128, "min": 0, "max": 0 }, "error_logger": 0, "couch_auth_cache": 0, "mango_sup": 0, "httpd_sup": 0, "couch_log_sup": 0, "couch_log_server": 0, "erl_prim_loader": 0, "global_group": 0, "standard_error_sup": 0, "erl_epmd": 0, "couch_epi_sup": 0, "couch_epi_functions_gen_couch_index": 0, "couch_epi_functions_gen_chttpd_auth": 0, "couch_epi_functions_gen_couch_db": 0, "couch_epi_functions_gen_global_changes": 0, "kernel_safe_sup": 0, "couch_epi_functions_gen_chttpd_handlers": 0, "couch_epi_functions_gen_chttpd": 0, "auth": 0, "code_server": 0, "rexi_buffer_couchdb@localhost": 0, "rexi_server_couchdb@localhost": 0, "mem3_sync_nodes": 0, "application_controller": 0, "httpc_sup": 0, "mochiweb_clock": 0, "mem3_sync": 0, "config_sup": 0, "mem3_sup": 0, "config": 0, "httpc_profile_sup": 0, "mem3_shards": 0, "global_changes_sup": 0, "inet_gethost_native_sup": 0, "httpc_manager": 0, "global_changes_server": 0, "httpc_handler_sup": 0, "runtime_tools_sup": 0, "mem3_nodes": 0, "net_kernel": 0, "user": 0, "ddoc_cache_sup": 0, "ftp_sup": 0, "rexi_sup": 0, "ddoc_cache_opener": 0, "rexi_server_sup": 0, "rexi_server_mon": 0, "rexi_server": 0, "ddoc_cache_lru": 0, "ioq_sup": 0, "chttpd_auth_cache_lru": 0, "memsup": 0, "mem3_events": 0, "disksup": 0, "ioq": 0, "couch_secondary_services": 0, "cpu_sup": 0, "couch_primary_services": 0, "couch_replicator_sup": 0, "ibrowse_sup": 0, "couch_replicator_manager": 0, "couch_replicator_job_sup": 0, "couch_uuids": 0, "rexi_buffer_mon": 0, "config_event": 0, "rexi_buffer_sup": 0, "ibrowse": 0, "couch_event_sup2": 0, "couch_event_server": 0, "couch_task_status": 0, "couch_event_os_sup": 0, "couch_sup": 0, "release_handler": 0, "couch_server": 0, "os_mon_sup": 0, "couch_replication": 0, "overload": 0, "couch_proc_manager": 0, "sasl_sup": 0, "couch_stats_sup": 0, "couch_stats_process_tracker": 0, "couch_plugin": 0, "alarm_handler": 0, "couch_os_daemons": 0, "tftp_sup": 0, "sasl_safe_sup": 0, "couch_stats_aggregator": 0, "timer_server": 0, "couch_httpd_vhost": 0, "folsom_sup": 0, "couch_index_sup": 0, "folsom_sample_slide_sup": 0, "couch_index_server": 0, "couch_httpd": 0, "rex": 0, "standard_error": 0, "init": 0, "net_sup": 0, "tls_connection_sup": 0, "chttpd_sup": 0, "kernel_sup": 0, "couch_external_manager": 0, "global_name_server": 0, "folsom_metrics_histogram_ets": 0, "file_server_2": 0, "couch_drv": 0, "ssl_sup": 0, "folsom_meter_timer_server": 0, "chttpd_auth_cache": 0, "setup_sup": 0, "inet_gethost_native": 0, "inet_db": 0, "inets_sup": 0, "ssl_manager": 0, "ssl_listen_tracker_sup": 0, "couch_compaction_daemon": 0, "chttpd": 0 }, "internal_replication_jobs": 0, "distribution": {} } ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
