I found that adjusting these RiakCS settings improved write speeds a little
for us:
{put_concurrency, 4},
{put_buffer_factor, 8},
As well as making sure the Riak vm.args included a high +zdbbl setting.
But even so, Riak CS has been really quite slow compared to other things
I've tried. However it has tended to work reasonably reliably and is easy
to maintain, so I went with it and it's been successful for us. I'd be
happier if write speeds were better though.
On 8 November 2014 01:41, David Meekin <david.mee...@autotrader.co.uk>
wrote:
> Hi,
>
>
>
> I’ve setup a test 4 node RiakCS cluster on HP BL460c hardware and I can’t
> seem to get S3 upload speeds above 2MB/s
>
> I’m connecting direct to RiackCS on one of the nodes so there is no load
> balancing software in place.
>
> I have also installed s3cmd locally onto one of the nodes and the speeds
> locally are the same.
>
> These 4 nodes also run a test CEPH cluster with RadosGW and s3 uploads to
> CEPH achieve 125MB/s
>
> Any help would be appreciated as I’m currently evaluating both CEPH and
> RiakCS.
>
>
>
> Configs below:
>
>
>
> %%%RIACK – app.config
>
> %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
>
> %% ex: ft=erlang ts=4 sw=4 et
>
> [
>
> %% Riak Client APIs config
>
> {riak_api, [
>
> {pb_backlog, 256},
>
> {pb, [ {"172.19.153.180", 8087 } ]}
>
> ]},
>
>
>
> %% Riak Core config
>
> {riak_core, [
>
>
>
> {default_bucket_props, [{allow_mult, true}]},
>
>
>
> {ring_state_dir, "/var/lib/riak/ring"},
>
>
>
> {http, [ {"172.19.153.180", 8098 } ]},
>
> {https, [{ "172.19.153.180", 8096 }]},
>
> {ssl, [
>
> {certfile, "/etc/riak/cert.pem"},
>
> {keyfile, "/etc/riak/key.pem"}
>
> ]},
>
>
>
> {handoff_port, 8099 },
>
>
>
> {dtrace_support, false},
>
>
>
> {platform_bin_dir, "/usr/sbin"},
>
> {platform_data_dir, "/var/lib/riak"},
>
> {platform_etc_dir, "/etc/riak"},
>
> {platform_lib_dir, "/usr/lib64/riak/lib"},
>
> {platform_log_dir, "/var/log/riak"}
>
> ]},
>
>
>
> %% Riak KV config
>
> {riak_kv, [
>
> {add_paths, ["/usr/lib64/riak-cs/lib/riak_cs-1.5.2/ebin"]},
>
> {storage_backend, riak_cs_kv_multi_backend},
>
> {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]},
>
> {multi_backend_default, be_default},
>
> {multi_backend, [
>
> {be_default, riak_kv_eleveldb_backend, [
>
> {max_open_files, 50},
>
> {data_root, "/var/lib/riak/leveldb"}
>
> ]},
>
> {be_blocks, riak_kv_bitcask_backend, [
>
> {data_root, "/var/lib/riak/bitcask"}
>
> ]}
>
> ]},
>
>
>
>
>
> {anti_entropy, {on, []}},
>
>
>
> {anti_entropy_build_limit, {1, 3600000}},
>
> {anti_entropy_expire, 604800000},
>
>
>
> {anti_entropy_concurrency, 2},
>
> {anti_entropy_tick, 15000},
>
>
>
> {anti_entropy_data_dir, "/var/lib/riak/anti_entropy"},
>
>
>
> {anti_entropy_leveldb_opts, [{write_buffer_size, 4194304},
>
> {max_open_files, 20}]},
>
>
>
> {mapred_name, "mapred"},
>
>
>
> {mapred_2i_pipe, true},
>
>
>
> {map_js_vm_count, 8 },
>
> {reduce_js_vm_count, 6 },
>
> {hook_js_vm_count, 2 },
>
>
>
> {js_max_vm_mem, 8},
>
>
>
> {js_thread_stack, 16},
>
>
>
> {http_url_encoding, on},
>
>
>
> {vnode_vclocks, true},
>
>
>
> {listkeys_backpressure, true},
>
>
>
> {fsm_limit, 50000},
>
>
>
> {object_format, v1}
>
> ]},
>
>
>
> %% Riak Search Config
>
> {riak_search, [
>
> {enabled, false}
>
> ]},
>
>
>
> %% Merge Index Config
>
> {merge_index, [
>
> {data_root, "/var/lib/riak/merge_index"},
>
>
>
> {buffer_rollover_size, 1048576},
>
>
>
> {max_compact_segments, 20}
>
> ]},
>
>
>
> %% Bitcask Config
>
> {bitcask, [
>
> {io_mode, erlang},
>
>
>
> {data_root, "/var/lib/riak/bitcask"}
>
> ]},
>
>
>
> %% eLevelDB Config
>
> {eleveldb, [
>
> {data_root, "/var/lib/riak/leveldb"}
>
> ]},
>
>
>
> %% Lager Config
>
> {lager, [
>
> {handlers, [
>
> {lager_file_backend, [
>
> {"/var/log/riak/error.log", error,
> 10485760, "$D0", 5},
>
> {"/var/log/riak/console.log", info,
> 10485760, "$D0", 5}
>
> ]}
>
> ] },
>
>
>
> {crash_log, "/var/log/riak/crash.log"},
>
>
>
> {crash_log_msg_size, 65536},
>
>
>
> {crash_log_size, 10485760},
>
>
>
> {crash_log_date, "$D0"},
>
>
>
> {crash_log_count, 5},
>
>
>
> {error_logger_redirect, true},
>
>
>
> {error_logger_hwm, 100}
>
> ]},
>
>
>
> %% riak_sysmon config
>
> {riak_sysmon, [
>
> {process_limit, 30},
>
> {port_limit, 2},
>
>
>
> {gc_ms_limit, 0},
>
> {heap_word_limit, 40111000},
>
>
>
> {busy_port, true},
>
> {busy_dist_port, true}
>
> ]},
>
>
>
> %% SASL config
>
> {sasl, [
>
> {sasl_error_logger, false}
>
> ]},
>
>
>
> %% riak_control config
>
> {riak_control, [
>
> {enabled, true},
>
>
>
> {auth, userlist},
>
>
>
> {userlist, [{"admin", "password"}
>
> ]},
>
>
>
> {admin, true}
>
> ]}
>
> ].
>
>
>
>
>
> %%%RiakCS – app.config
>
> %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
>
> %% ex: ts=4 sw=4 et
>
> [
>
> %% Riak CS configuration
>
> {riak_cs, [
>
> %% == Basic Configuration ==
>
>
>
> %% Riak CS http/https port and IP address to listen at
>
> %% for object storage activity
>
> {cs_ip, "172.19.153.180"},
>
> {cs_port, 8080 } ,
>
> {riak_ip, "172.19.153.180"},
>
> {riak_pb_port, 8087 } ,
>
> {stanchion_ip, "172.19.153.180"},
>
> {stanchion_port, 8085 },
>
> {stanchion_ssl, false },
>
> {anonymous_user_creation, true},
>
> {admin_key, "_UJJOXXQEXW3XAWUGSQD"},
>
> {admin_secret, "8bkzcxZDIKsmjJUhEEJkpcM6g5_X7gSazryBGA=="},
>
>
>
> {cs_root_host, "s3.amazonaws.com"},
>
> {connection_pools,
>
> [
>
> {request_pool, {128, 0} },
>
> {bucket_list_pool, {5, 0} }
>
> ]},
>
> {rewrite_module, riak_cs_s3_rewrite },
>
> {auth_module, riak_cs_s3_auth },
>
> {fold_objects_for_list_keys, true},
>
>
>
> {n_val_1_get_requests, true},
>
> {cs_version, 10300 },
>
> {access_log_flush_factor, 1},
>
> {access_log_flush_size, 1000000},
>
> {access_archive_period, 3600},
>
> {access_archiver_max_backlog, 2},
>
> {storage_schedule, []},
>
> {storage_archive_period, 86400},
>
> {usage_request_limit, 744},
>
> {leeway_seconds, 86400},
>
> {gc_interval, 900},
>
> {gc_retry_interval, 21600},
>
> {gc_paginated_indexes, true},
>
>
>
> {trust_x_forwarded_for, false},
>
> {dtrace_support, false}
>
>
>
> ]},
>
>
>
> {webmachine, [
>
> {server_name, "Riak CS"},
>
>
>
> {log_handlers, [
>
> {webmachine_log_handler,
> ["/var/log/riak-cs"]},
>
> {riak_cs_access_log_handler, []}
>
> ]}
>
> ]},
>
>
>
> {lager, [
>
> {handlers, [
>
> {lager_console_backend, info},
>
> {lager_file_backend, [{file,
> "/var/log/riak-cs/error.log"},
>
> {level, error},
>
> {size, 10485760},
>
> {date, "$D0"},
>
> {count, 5}]},
>
> {lager_file_backend, [{file,
> "/var/log/riak-cs/console.log"},
>
> {level, info},
>
> {size, 10485760},
>
> {date, "$D0"},
>
> {count, 5}]}
>
> ]},
>
> {crash_log, "/var/log/riak-cs/crash.log"},
>
> {crash_log_msg_size, 65536},
>
>
>
> {crash_log_size, 10485760},
>
>
>
> {crash_log_date, "$D0"},
>
>
>
> {crash_log_count, 5},
>
>
>
> {error_logger_redirect, true}
>
> ]},
>
>
>
> {sasl, [
>
> {sasl_error_logger, false}
>
> ]}
>
> ].
>
>
>
> Thanks,
>
>
>
> Dave
>
> Tel: +441616699154 Mob: +44 7525 405543
>
>
>
> ________________________________________
>
> This e-mail is sent on behalf of Auto Trader Group Limited, Registered
> Office: Auto Trader House, Cutbush Park Industrial Estate, Danehill, Lower
> Earley, Reading, Berkshire, RG6 4UT (Registered in England No. 4768833).
> This email and any files transmitted with it are confidential and may be
> legally privileged, and intended solely for the use of the individual or
> entity to whom they are addressed. If you have received this email in error
> please notify the sender. This email message has been swept for the
> presence of computer viruses.
>
> _______________________________________________
> riak-users mailing list
> riak-users@lists.basho.com
> http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
>
>
--
Turning and turning in the widening gyre
The falcon cannot hear the falconer
Things fall apart; the center cannot hold
Mere anarchy is loosed upon the world
_______________________________________________
riak-users mailing list
riak-users@lists.basho.com
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
