Release the new website - add load test page - add global cluster page
Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/7e299703 Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/7e299703 Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/7e299703 Branch: refs/heads/asf-site Commit: 7e299703e11c9a93e1f810cc6fdea6a14b98664c Parents: 6e9a8fa Author: Sijie Guo <[email protected]> Authored: Fri Dec 16 03:20:04 2016 -0800 Committer: Sijie Guo <[email protected]> Committed: Fri Dec 16 03:20:04 2016 -0800 ---------------------------------------------------------------------- content/community/index.html | 15 +- content/docs/latest/admin_guide/bookkeeper.html | 40 +- content/docs/latest/admin_guide/hardware.html | 40 +- content/docs/latest/admin_guide/loadtest.html | 602 ++++++++++++++++++ content/docs/latest/admin_guide/main.html | 40 +- content/docs/latest/admin_guide/monitoring.html | 40 +- content/docs/latest/admin_guide/operations.html | 40 +- .../docs/latest/admin_guide/performance.html | 40 +- content/docs/latest/admin_guide/vagrant.rst | 4 +- content/docs/latest/admin_guide/zookeeper.html | 40 +- content/docs/latest/basics/introduction.html | 30 +- content/docs/latest/deployment/cluster.html | 45 +- content/docs/latest/deployment/docker.html | 16 +- .../docs/latest/deployment/global-cluster.html | 615 +++++++++++++++++++ content/docs/latest/index.html | 16 +- content/docs/latest/performance/main.html | 16 +- content/docs/latest/start/building.html | 16 +- content/docs/latest/start/download.html | 16 +- content/docs/latest/start/quickstart.html | 20 +- .../latest/tutorials/analytics-mapreduce.html | 16 +- content/docs/latest/tutorials/basic-1.html | 16 +- content/docs/latest/tutorials/basic-2.html | 16 +- content/docs/latest/tutorials/basic-3.html | 16 +- content/docs/latest/tutorials/basic-4.html | 16 +- content/docs/latest/tutorials/basic-5.html | 16 +- content/docs/latest/tutorials/basic-6.html | 16 +- content/docs/latest/tutorials/main.html | 16 +- content/docs/latest/tutorials/messaging-1.html | 16 +- content/docs/latest/tutorials/messaging-2.html | 16 +- content/docs/latest/tutorials/messaging-3.html | 16 +- content/docs/latest/tutorials/messaging-4.html | 16 +- content/docs/latest/tutorials/messaging-5.html | 16 +- .../tutorials/replicatedstatemachines.html | 16 +- content/docs/latest/user_guide/api/core.html | 16 +- content/docs/latest/user_guide/api/main.html | 16 +- .../docs/latest/user_guide/api/practice.html | 16 +- content/docs/latest/user_guide/api/proxy.html | 16 +- .../latest/user_guide/architecture/main.html | 16 +- .../latest/user_guide/configuration/client.html | 16 +- .../latest/user_guide/configuration/core.html | 16 +- .../latest/user_guide/configuration/main.html | 16 +- .../latest/user_guide/configuration/perlog.html | 16 +- .../latest/user_guide/configuration/proxy.html | 16 +- .../latest/user_guide/considerations/main.html | 16 +- content/docs/latest/user_guide/design/main.html | 16 +- .../user_guide/globalreplicatedlog/main.html | 16 +- .../latest/user_guide/implementation/core.html | 16 +- .../latest/user_guide/implementation/main.html | 16 +- .../user_guide/implementation/storage.html | 16 +- .../user_guide/implementation/writeproxy.html | 16 +- content/docs/latest/user_guide/main.html | 16 +- .../latest/user_guide/references/features.html | 16 +- .../docs/latest/user_guide/references/main.html | 16 +- .../latest/user_guide/references/metrics.html | 16 +- content/images/distributedlog_logo_m.png | Bin 0 -> 9172 bytes 55 files changed, 2137 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/community/index.html ---------------------------------------------------------------------- diff --git a/content/community/index.html b/content/community/index.html index 9a099b7..c7da007 100644 --- a/content/community/index.html +++ b/content/community/index.html @@ -183,8 +183,21 @@ </thead> <tr> <td> + <strong>user</strong>@@distributedlog.incubator.apache.org<br /> + <small>User related discussions</small> + </td> + <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Subscribe</a></td> + <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Subscribe</a></td> + <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Unsubscribe</a></td> + <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Post</a></td> + <td class="text-center"> + <a href="http://mail-archives.apache.org/mod_mbox/incubator-distributedlog-user/";>Archives</a> <br /> + </td> + </tr> + <tr> + <td> <strong>dev</strong>@@distributedlog.incubator.apache.org<br /> - <small>Users and Development related discussions</small> + <small>Development related discussions</small> </td> <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Subscribe</a></td> <td class="text-center"><i class="fa fa-pencil-square-o"></i> <a href="mailto:[email protected]";>Subscribe</a></td> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/bookkeeper.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/bookkeeper.html b/content/docs/latest/admin_guide/bookkeeper.html index ee192d6..e985f8e 100644 --- a/content/docs/latest/admin_guide/bookkeeper.html +++ b/content/docs/latest/admin_guide/bookkeeper.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/bookkeeper.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/bookkeeper.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/hardware.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/hardware.html b/content/docs/latest/admin_guide/hardware.html index 0eeacb3..53aecb5 100644 --- a/content/docs/latest/admin_guide/hardware.html +++ b/content/docs/latest/admin_guide/hardware.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/hardware.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/hardware.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class="active"> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/loadtest.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/loadtest.html b/content/docs/latest/admin_guide/loadtest.html new file mode 100644 index 0000000..c0b4113 --- /dev/null +++ b/content/docs/latest/admin_guide/loadtest.html @@ -0,0 +1,602 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache DistributedLog (incubating)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/latest/styles/site.css"> + <link rel="stylesheet" href="/docs/latest/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/latest/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/loadtest.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/latest/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/latest/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/latest/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/latest/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/latest/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/latest/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/latest/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/latest/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/latest/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/latest/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/latest/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/latest/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/latest/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/latest/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/latest/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/latest/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/latest/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/latest/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/latest/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/latest/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/latest/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<div class="row"> + <!-- Sub Navigation --> + <div class="col-sm-3"> + <ul id="sub-nav"> + + + + + <li><a href="/docs/latest/admin_guide/main.html" class="">Admin Guide</a> + + <ul> + + + <li> + <a href="/docs/latest/admin_guide/operations.html" class=""> + Operations + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/deployment/cluster.html" class=""> + Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/performance.html" class=""> + Performance Tuning + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/loadtest.html" class="active"> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/hardware.html" class=""> + Hardware + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/monitoring.html" class=""> + Monitoring + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/zookeeper.html" class=""> + ZooKeeper + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/bookkeeper.html" class=""> + BookKeeper + </a> + + <ul> + + </ul> + + </li> + + </ul> + + </li> + + </ul> + </div> + <!-- Main --> + <div class="col-sm-9"> + <!-- Top anchor --> + <a href="#top"></a> + + <!-- Breadcrumbs above the main heading --> + <ol class="breadcrumb"> + + + + + + + + + + + + + <li><a href="/docs/latest/admin_guide/main.html">Admin Guide</a></li> + + + <li class="active">Load Test</li> + </ol> + + <div class="text"> + <!-- Content --> + <div class="contents topic" id="load-test"> +<p class="topic-title first">Load Test</p> +<ul class="simple"> +<li><a class="reference internal" href="#id1" id="id2">Load Test</a><ul> +<li><a class="reference internal" href="#overview" id="id3">Overview</a></li> +<li><a class="reference internal" href="#running-dbench" id="id4">Running Dbench</a></li> +<li><a class="reference internal" href="#write-to-proxy-with-thin-client" id="id5">Write to Proxy with Thin Client</a></li> +<li><a class="reference internal" href="#write-to-bookkeeper-with-core-library" id="id6">Write to BookKeeper with Core Library</a></li> +<li><a class="reference internal" href="#read-from-bookkeeper-with-core-library" id="id7">Read from BookKeeper with Core Library</a></li> +</ul> +</li> +</ul> +</div> +<div class="section" id="id1"> +<h2><a class="toc-backref" href="#id2">Load Test</a></h2> +<div class="section" id="overview"> +<h3><a class="toc-backref" href="#id3">Overview</a></h3> +<p>Under distributedlog-benchmark you will find a set of applications intended for generating large amounts of load in a distributedlog custer. These applications are suitable for load testing, performance testing, benchmarking, or even simply smoke testing a distributedlog cluster.</p> +<p>The dbench script can run in several modes:</p> +<ol class="arabic simple"> +<li>bkwrite - Benchmark the distributedlog write path using the core library</li> +<li>write - Benchmark the distributedlog write path, via write proxy, using the thin client</li> +<li>read - Benchmark the distributedlog read path using the core library</li> +</ol> +</div> +<div class="section" id="running-dbench"> +<h3><a class="toc-backref" href="#id4">Running Dbench</a></h3> +<p>The distributedlog-benchmark binary dbench is intended to be run simultaneously from many machines with identical settings. Together, all instances of dbench comprise a benchmark job. How you launch a benchmark job will depend on your operating environment. We recommend using a cluster scheduler like aurora or kubernetes to simplify the process, but tools like capistrano can also simplify this process greatly.</p> +<p>The benchmark script can be found at</p> +<pre class="literal-block"> +distributedlog-benchmark/bin/dbench +</pre> +<p>Arguments may be passed to this script via environment variables. The available arguments depend on the execution mode. For an up to date list, check the script itself.</p> +</div> +<div class="section" id="write-to-proxy-with-thin-client"> +<h3><a class="toc-backref" href="#id5">Write to Proxy with Thin Client</a></h3> +<p>The proxy write test (mode = <cite>write</cite>) can be used to send writes to a proxy cluster to be written to a set of streams.</p> +<p>For example to use the proxy write test to generate 10000 requests per second across 10 streams using 50 machines, run the following command on each machine.</p> +<pre class="literal-block"> +STREAM_NAME_PREFIX=loadtest_ +BENCHMARK_DURATION=3600 # seconds +DL_NAMESPACE=<dl namespace> +NUM_STREAMS=10 +INITIAL_RATE=200 +distributedlog-benchmark/bin/dbench write +</pre> +</div> +<div class="section" id="write-to-bookkeeper-with-core-library"> +<h3><a class="toc-backref" href="#id6">Write to BookKeeper with Core Library</a></h3> +<p>The core library write test (mode = <cite>bkwrite</cite>) can be used to send writes to directly to bookkeeper using the core library.</p> +<p>For example to use the core library write test to generate 100MBps across 10 streams using 100 machines, run the following command on each machine.</p> +<pre class="literal-block"> +STREAM_NAME_PREFIX=loadtest_ +BENCHMARK_DURATION=3600 # seconds +DL_NAMESPACE=<dl namespace> +NUM_STREAMS=10 +INITIAL_RATE=1024 +MSG_SIZE=1024 +distributedlog-benchmark/bin/dbench bkwrite +</pre> +</div> +<div class="section" id="read-from-bookkeeper-with-core-library"> +<h3><a class="toc-backref" href="#id7">Read from BookKeeper with Core Library</a></h3> +<p>The core library read test (mode = <cite>read</cite>) can be used to read directly from bookkeeper using the core library.</p> +<p>For example to use the core library read test to read from 10 streams on 100 instances, run the following command on each machine.</p> +<pre class="literal-block"> +STREAM_NAME_PREFIX=loadtest_ +BENCHMARK_DURATION=3600 # seconds +DL_NAMESPACE=<dl namespace> +MAX_STREAM_ID=9 +NUM_READERS_PER_STREAM=5 +TRUNCATION_INTERVAL=60 # seconds +distributedlog-benchmark/bin/dbench read +</pre> +</div> +</div> + + + </div> + </div> +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/latest/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/main.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/main.html b/content/docs/latest/admin_guide/main.html index 27cb277..28e2c0d 100644 --- a/content/docs/latest/admin_guide/main.html +++ b/content/docs/latest/admin_guide/main.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/main.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/main.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -336,6 +348,18 @@ <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -348,6 +372,18 @@ <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/monitoring.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/monitoring.html b/content/docs/latest/admin_guide/monitoring.html index ae57f2a..1d91a38 100644 --- a/content/docs/latest/admin_guide/monitoring.html +++ b/content/docs/latest/admin_guide/monitoring.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/monitoring.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/monitoring.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/operations.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/operations.html b/content/docs/latest/admin_guide/operations.html index c9f4ac6..065eeb8 100644 --- a/content/docs/latest/admin_guide/operations.html +++ b/content/docs/latest/admin_guide/operations.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/operations.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/operations.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/performance.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/performance.html b/content/docs/latest/admin_guide/performance.html index 526ea1a..dbe66c6 100644 --- a/content/docs/latest/admin_guide/performance.html +++ b/content/docs/latest/admin_guide/performance.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/performance.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/performance.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class="active"> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/vagrant.rst ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/vagrant.rst b/content/docs/latest/admin_guide/vagrant.rst index 70e6fb0..e4ed574 100644 --- a/content/docs/latest/admin_guide/vagrant.rst +++ b/content/docs/latest/admin_guide/vagrant.rst @@ -13,6 +13,6 @@ Steps 1. Create a snapshot using ./scripts/snapshot 2. Run vagrant up from the root directory of the enlistment 3. Vagrant brings up a zookeepers with machine names: zk1,zk2, .... with IP addresses 192.168.50.11,192.168.50.12,.... -4. Vagrant brings the bookies with machine names: node1,node2, ... with IP addresses 192.168.50.51,192.168.50.52,.... -5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script. +4. Vagrant brings the bookies with machine names: bk1,bk2, ... with IP addresses 192.168.50.51,192.168.50.52,.... +5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script. 6. If you want to run the client on the host machine, please add these node names and their IP addresses to the /etc/hosts file on the host. http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/admin_guide/zookeeper.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/admin_guide/zookeeper.html b/content/docs/latest/admin_guide/zookeeper.html index e3b3c47..8647e62 100644 --- a/content/docs/latest/admin_guide/zookeeper.html +++ b/content/docs/latest/admin_guide/zookeeper.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/admin_guide/zookeeper.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/admin_guide/zookeeper.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/basics/introduction.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/basics/introduction.html b/content/docs/latest/basics/introduction.html index b5210dc..3a63539 100644 --- a/content/docs/latest/basics/introduction.html +++ b/content/docs/latest/basics/introduction.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/basics/introduction.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/basics/introduction.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -638,9 +650,9 @@ might have different requirements on tracking and coordinating positions. It is to get it right with a single approach. For example, distributed databases might store the reader positions along with SSTables, so they would resume applying transactions from the positions stored in SSTables. Tracking reader positions could easily be done -in application level using various stores (e.g. ZooKeeper, FileSystem, or Key/Value Stores).</p> +in application level using various stores (e.g. ZooKeeper, file system, or key/value stores).</p> <p>The log records could be cached in a service tier called <em>Read Proxy</em>, to serve -large number of readers. Details on <strong>Fan-in and Fan-out</strong> can be found further into this doc.</p> +a large number of readers.</p> </div> <div class="section" id="fan-in-and-fan-out"> <h3><a class="toc-backref" href="#id8">Fan-in and Fan-out</a></h3> @@ -654,14 +666,14 @@ thousands of readers are consuming a same log stream.</p> </div> <div class="section" id="guarantees"> <h3><a class="toc-backref" href="#id9">Guarantees</a></h3> -<p>At a high-level, DistributedLog gives the following guarantees:</p> +<p>At a high level, DistributedLog gives the following guarantees:</p> <ul class="simple"> <li>Records written by a writer to a log will be appended in the order they are written. That is, if a record <em>R1</em> is written by same writer as a record <em>R2</em>, <em>R1</em> will have a smaller sequence number than <em>R2</em>.</li> -<li>Readers will see records in same order they were written to the log.</li> -<li>All records were persisted on disks before acknowledges, to gurantee durability.</li> -<li>For a log with replication factor of N, DistributedLog tolerates up to N-1 server failures without losing any records appended to the log.</li> +<li>Readers see records in the same order they were written to the log.</li> +<li>All records are persisted on disk before acknowledgments, to gurantee durability.</li> +<li>For a log with replication factor of N, DistributedLog tolerates up to N-1 server failures without losing any records.</li> </ul> -<p>More details on these guarantees are given in the design section of this documentation.</p> +<p>More details on these guarantees are given in the [design section](<a class="reference external" href="http://distributedlog.incubator.apache.org/docs/latest/user_guide/design/main.html";>http://distributedlog.incubator.apache.org/docs/latest/user_guide/design/main.html</a>).</p> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/deployment/cluster.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/deployment/cluster.html b/content/docs/latest/deployment/cluster.html index 74b0f99..218b108 100644 --- a/content/docs/latest/deployment/cluster.html +++ b/content/docs/latest/deployment/cluster.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/deployment/cluster.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/deployment/cluster.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> @@ -358,6 +370,18 @@ under the License. <li> + <a href="/docs/latest/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/performance.html" class=""> Performance Tuning </a> @@ -370,6 +394,18 @@ under the License. <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> <a href="/docs/latest/admin_guide/hardware.html" class=""> Hardware </a> @@ -493,8 +529,9 @@ To run a cluster with DistributedLog, you need a Zookeeper cluster and a Bookkee </span></code></pre></td></tr></table></div></figure><p>Or run <cite>./scripts/snapshot</cite> to build the release packages from current source. The released packages contain the binaries for running <cite>distributedlog-service</cite>, <cite>distributedlog-benchmark</cite> and <cite>distributedlog-tutorials</cite>.</p> -<p>NOTE: we run following instructions from distributedlog source code after running <cite>mvn clean install</cite>. -And assume <cite>DL_HOME</cite> is the directory of distributedlog source.</p> +<p>NOTE: we run the following instructions from distributedlog source code after +running <cite>mvn clean install</cite>. And assume <cite>DL_HOME</cite> is the directory of +distributedlog source.</p> </div> <div class="section" id="zookeeper"> <h3><a class="toc-backref" href="#id3">Zookeeper</a></h3> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/deployment/docker.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/deployment/docker.html b/content/docs/latest/deployment/docker.html index f50e4ea..3652dd0 100644 --- a/content/docs/latest/deployment/docker.html +++ b/content/docs/latest/deployment/docker.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/deployment/docker.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/deployment/docker.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/deployment/global-cluster.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/deployment/global-cluster.html b/content/docs/latest/deployment/global-cluster.html new file mode 100644 index 0000000..bda3cb5 --- /dev/null +++ b/content/docs/latest/deployment/global-cluster.html @@ -0,0 +1,615 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Global Cluster Setup</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/latest/styles/site.css"> + <link rel="stylesheet" href="/docs/latest/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/latest/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/deployment/global-cluster.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/latest/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/latest/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/latest/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/latest/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/latest/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/latest/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/latest/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/latest/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/latest/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/latest/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/latest/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/latest/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/latest/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/latest/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/latest/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/latest/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/latest/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/latest/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/latest/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/latest/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/latest/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/latest/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/latest/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/latest/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<div class="row"> + <!-- Sub Navigation --> + <div class="col-sm-3"> + <ul id="sub-nav"> + + + + + <li><a href="/docs/latest/admin_guide/main.html" class="">Admin Guide</a> + + <ul> + + + <li> + <a href="/docs/latest/admin_guide/operations.html" class=""> + Operations + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/deployment/cluster.html" class=""> + Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/deployment/global-cluster.html" class="active"> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/performance.html" class=""> + Performance Tuning + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/hardware.html" class=""> + Hardware + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/monitoring.html" class=""> + Monitoring + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/zookeeper.html" class=""> + ZooKeeper + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/latest/admin_guide/bookkeeper.html" class=""> + BookKeeper + </a> + + <ul> + + </ul> + + </li> + + </ul> + + </li> + + </ul> + </div> + <!-- Main --> + <div class="col-sm-9"> + <!-- Top anchor --> + <a href="#top"></a> + + <!-- Breadcrumbs above the main heading --> + <ol class="breadcrumb"> + + + + + + + + + + + + + <li><a href="/docs/latest/admin_guide/main.html">Admin Guide</a></li> + + + <li class="active">Global Cluster Setup</li> + </ol> + + <div class="text"> + <!-- Content --> + <div class="contents topic" id="this-page-provides-instructions-on-how-to-run-distributedlog-across-multiple-regions"> +<p class="topic-title first">This page provides instructions on how to run <strong>DistributedLog</strong> across multiple regions.</p> +<ul class="simple"> +<li><a class="reference internal" href="#cluster-setup-deployment" id="id1">Cluster Setup & Deployment</a><ul> +<li><a class="reference internal" href="#global-zookeeper" id="id2">Global Zookeeper</a></li> +<li><a class="reference internal" href="#distributedlog-configuration" id="id3">DistributedLog Configuration</a><ul> +<li><a class="reference internal" href="#placement-policy" id="id4">Placement Policy</a></li> +<li><a class="reference internal" href="#connection-timeouts" id="id5">Connection Timeouts</a></li> +<li><a class="reference internal" href="#quorum-size" id="id6">Quorum Size</a></li> +</ul> +</li> +<li><a class="reference internal" href="#client-configuration" id="id7">Client Configuration</a></li> +</ul> +</li> +<li><a class="reference internal" href="#additional-steps" id="id8">Additional Steps</a></li> +</ul> +</div> +<div class="section" id="cluster-setup-deployment"> +<h2><a class="toc-backref" href="#id1">Cluster Setup & Deployment</a></h2> +<p>Setting up <a class="reference external" href="../user_guide/globalreplicatedlog/main">globally replicated DistributedLog</a> is very similar to setting up local DistributedLog. +The most important change is use a ZooKeeper cluster configured across multiple-regions. Once set up, DistributedLog +and BookKeeper are configured to use the global ZK cluster for all metadata storage, and the system will more or +less work. The remaining steps are necessary to ensure things like durability in the face of total region failure.</p> +<p>The key differences with standard cluster setup are summarized below:</p> +<ul class="simple"> +<li>The zookeeper cluster must be running across all of the target regions, say A, B, C.</li> +<li>Region aware placement policy and a few other options must be configured in DL config.</li> +<li>DistributedLog clients should be configured to talk to all regions.</li> +</ul> +<p>We elaborate on these steps in the following sections.</p> +<div class="section" id="global-zookeeper"> +<h3><a class="toc-backref" href="#id2">Global Zookeeper</a></h3> +<p>When defining your server and participant lists in zookeeper configuration, a sufficient number of nodes from each +region must be included.</p> +<p>Please consult the ZooKeeper documentation for detailed Zookeeper setup instructions.</p> +</div> +<div class="section" id="distributedlog-configuration"> +<h3><a class="toc-backref" href="#id3">DistributedLog Configuration</a></h3> +<p>In multi-region DistributedLog several DL config changes are needed.</p> +<div class="section" id="placement-policy"> +<h4><a class="toc-backref" href="#id4">Placement Policy</a></h4> +<p>The region-aware placement policy must be configured. Below, it is configured to place replicas across 3 regions, A, B, and C.</p> +<pre class="literal-block"> +# placement policy +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RegionAwareEnsemblePlacementPolicy +bkc.reppRegionsToWrite=A;B;C +bkc.reppMinimumRegionsForDurability=2 +bkc.reppEnableDurabilityEnforcementInReplace=true +bkc.reppEnableValidation=true +</pre> +</div> +<div class="section" id="connection-timeouts"> +<h4><a class="toc-backref" href="#id5">Connection Timeouts</a></h4> +<p>In global replicated mode, the proxy nodes will be writing to the entire ensemble, which exists in multiple regions. +If cross-region latency is higher than local region latency (i.e. if its truly cross-region) then it is advisable to +use a higher BookKeeper client connection tiemout.</p> +<pre class="literal-block"> +# setting connect timeout to 1 second for global cluster +bkc.connectTimeoutMillis=1000 +</pre> +</div> +<div class="section" id="quorum-size"> +<h4><a class="toc-backref" href="#id6">Quorum Size</a></h4> +<p>It is advisable to run with a larger ensemble to ensure cluster health in the event of region loss (the ensemble +will be split across all regions).</p> +<p>The values of these settings will depend on your operational and durability requirements.</p> +<pre class="literal-block"> +ensemble-size=9 +write-quorum-size=9 +ack-quorum-size=5 +</pre> +</div> +</div> +<div class="section" id="client-configuration"> +<h3><a class="toc-backref" href="#id7">Client Configuration</a></h3> +<p>Although not required, it is recommended to configure the write client to use all available regions. Several methods +in DistributedLogClientBuilder can be used to achieve this.</p> +<figure class="code"><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class="line-number">1</span> +<span class="line-number">2</span> +</pre></td><td class="code"><pre><code class="java"><span class="line"><span></span><span class="n">DistributedLogClientBuilder</span><span class="o">.</span><span class="na">serverSets</span> +</span><span class="line"><span class="n">DistributedLogClientBuilder</span><span class="o">.</span><span class="na">finagleNameStrs</span> +</span></code></pre></td></tr></table></div></figure></div> +</div> +<div class="section" id="additional-steps"> +<h2><a class="toc-backref" href="#id8">Additional Steps</a></h2> +<p>Other clients settings may need to be tuned - for example in the write client, timeouts will likely need to be +increased.</p> +<p>Aside from this however, cluster setup is exactly the same as <a class="reference external" href="cluster">single region setup</a>.</p> +</div> + + + </div> + </div> +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/latest/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/index.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/index.html b/content/docs/latest/index.html index 4a41780..5807eab 100644 --- a/content/docs/latest/index.html +++ b/content/docs/latest/index.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/performance/main.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/performance/main.html b/content/docs/latest/performance/main.html index e773a8a..77253e2 100644 --- a/content/docs/latest/performance/main.html +++ b/content/docs/latest/performance/main.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/performance/main.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/performance/main.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/7e299703/content/docs/latest/start/building.html ---------------------------------------------------------------------- diff --git a/content/docs/latest/start/building.html b/content/docs/latest/start/building.html index 2472601..2cdb9b1 100644 --- a/content/docs/latest/start/building.html +++ b/content/docs/latest/start/building.html @@ -15,8 +15,8 @@ <!-- JQuery --> <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> <script src="/docs/latest/js/bootstrap.min.js"></script> - <link rel="canonical" href="/docs/latest/start/building.html" data-proofer-ignore> - <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="/docs/latest/feed.xml"> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/latest/start/building.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/latest/feed.xml";> <!-- Font Awesome --> <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> <!-- Google Analytics --> @@ -113,6 +113,12 @@ </li> <li> + <a href="/docs/latest/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> <a href="/docs/latest/deployment/docker.html"> Docker </a> @@ -202,6 +208,12 @@ </li> <li> + <a href="/docs/latest/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> <a href="/docs/latest/admin_guide/hardware.html"> Hardware </a>
