http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/49afadb4/docs/0.4.0-incubating/admin_guide/monitoring.html ---------------------------------------------------------------------- diff --git a/docs/0.4.0-incubating/admin_guide/monitoring.html b/docs/0.4.0-incubating/admin_guide/monitoring.html new file mode 100644 index 0000000..82a1305 --- /dev/null +++ b/docs/0.4.0-incubating/admin_guide/monitoring.html @@ -0,0 +1,866 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache DistributedLog (incubating)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/admin_guide/monitoring.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<div class="row"> + <!-- Sub Navigation --> + <div class="col-sm-3"> + <ul id="sub-nav"> + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html" class="">Admin Guide</a> + + <ul> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html" class=""> + Operations + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html" class=""> + Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html" class=""> + Performance Tuning + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html" class=""> + Hardware + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html" class="active"> + Monitoring + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html" class=""> + ZooKeeper + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html" class=""> + BookKeeper + </a> + + <ul> + + </ul> + + </li> + + </ul> + + </li> + + </ul> + </div> + <!-- Main --> + <div class="col-sm-9"> + <!-- Top anchor --> + <a href="#top"></a> + + <!-- Breadcrumbs above the main heading --> + <ol class="breadcrumb"> + + + + + + + + + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html">Admin Guide</a></li> + + + <li class="active">Monitoring</li> + </ol> + + <div class="text"> + <!-- Content --> + <div class="contents topic" id="monitoring"> +<p class="topic-title first">Monitoring</p> +<ul class="simple"> +<li><a class="reference internal" href="#id1" id="id4">Monitoring</a><ul> +<li><a class="reference internal" href="#stats-provider" id="id5">Stats Provider</a><ul> +<li><a class="reference internal" href="#stats-logger" id="id6">Stats Logger</a><ul> +<li><a class="reference internal" href="#counters" id="id7">Counters</a></li> +<li><a class="reference internal" href="#gauges" id="id8">Gauges</a></li> +<li><a class="reference internal" href="#metrics-opstats" id="id9">Metrics (OpStats)</a></li> +</ul> +</li> +</ul> +</li> +<li><a class="reference internal" href="#available-stats-providers" id="id10">Available Stats Providers</a><ul> +<li><a class="reference internal" href="#twitter-science-stats" id="id11">Twitter Science Stats</a></li> +<li><a class="reference internal" href="#twitter-ostrich-stats" id="id12">Twitter Ostrich Stats</a></li> +<li><a class="reference internal" href="#twitter-finagle-metrics" id="id13">Twitter Finagle Metrics</a></li> +<li><a class="reference internal" href="#codahale-metrics" id="id14">Codahale Metrics</a></li> +</ul> +</li> +<li><a class="reference internal" href="#enable-stats-provider-on-bookie-servers" id="id15">Enable Stats Provider on Bookie Servers</a></li> +<li><a class="reference internal" href="#metrics" id="id16">Metrics</a></li> +</ul> +</li> +</ul> +</div> +<div class="section" id="id1"> +<h2><a class="toc-backref" href="#id4">Monitoring</a></h2> +<p>DistributedLog uses the stats library provided by Apache BookKeeper for reporting metrics in +both the server and the client. This can be configured to report stats using pluggable stats +provider to integrate with your monitoring system.</p> +<div class="section" id="stats-provider"> +<h3><a class="toc-backref" href="#id5">Stats Provider</a></h3> +<p><cite>StatsProvider</cite> is a provider that provides different kinds of stats logger for different scopes. +The provider is also responsible for reporting its managed metrics.</p> +<pre class="literal-block"> +// Create the stats provider +StatsProvider statsProvider = ...; +// Start the stats provider +statsProvider.start(conf); +// Stop the stats provider +statsProvider.stop(); +</pre> +<div class="section" id="stats-logger"> +<h4><a class="toc-backref" href="#id6">Stats Logger</a></h4> +<p>A scoped <cite>StatsLogger</cite> is a stats logger that records 3 kinds of statistics +under a given <cite>scope</cite>.</p> +<p>A <cite>StatsLogger</cite> could be either created by obtaining from stats provider with +the scope name:</p> +<pre class="literal-block"> +StatsProvider statsProvider = ...; +StatsLogger statsLogger = statsProvider.scope("test-scope"); +</pre> +<p>Or created by obtaining from a stats logger with a sub scope name:</p> +<pre class="literal-block"> +StatsLogger rootStatsLogger = ...; +StatsLogger subStatsLogger = rootStatsLogger.scope("sub-scope"); +</pre> +<p>All the metrics in a stats provider are managed in a hierarchical of scopes.</p> +<pre class="literal-block"> +// all stats recorded by `rootStatsLogger` are under 'root' +StatsLogger rootStatsLogger = statsProvider.scope("root"); +// all stats recorded by 'subStatsLogger1` are under 'root/scope1' +StatsLogger subStatsLogger1 = statsProvider.scope("scope1"); +// all stats recorded by 'subStatsLogger2` are under 'root/scope2' +StatsLogger subStatsLogger2 = statsProvider.scope("scope2"); +</pre> +<div class="section" id="counters"> +<h5><a class="toc-backref" href="#id7">Counters</a></h5> +<p>A <cite>Counter</cite> is a cumulative metric that represents a single numerical value. A <strong>counter</strong> +is typically used to count requests served, tasks completed, errors occurred, etc. Counters +should not be used to expose current counts of items whose number can also go down, e.g. +the number of currently running tasks. Use <cite>Gauges</cite> for this use case.</p> +<p>To change a counter, use:</p> +<pre class="literal-block"> +StatsLogger statsLogger = ...; +Counter births = statsLogger.getCounter("births"); +// increment the counter +births.inc(); +// decrement the counter +births.dec(); +// change the counter by delta +births.add(-10); +// reset the counter +births.reset(); +</pre> +</div> +<div class="section" id="gauges"> +<h5><a class="toc-backref" href="#id8">Gauges</a></h5> +<p>A <cite>Gauge</cite> is a metric that represents a single numerical value that can arbitrarily go up and down.</p> +<p>Gauges are typically used for measured values like temperatures or current memory usage, but also +"counts" that can go up and down, like the number of running tasks.</p> +<p>To define a gauge, stick the following code somewhere in the initialization:</p> +<pre class="literal-block"> +final AtomicLong numPendingRequests = new AtomicLong(0L); +StatsLogger statsLogger = ...; +statsLogger.registerGauge( + "num_pending_requests", + new Gauge<Number>() { + @Override + public Number getDefaultValue() { + return 0; + } + @Override + public Number getSample() { + return numPendingRequests.get(); + } + }); +</pre> +<p>The gauge must always return a numerical value when sampling.</p> +</div> +<div class="section" id="metrics-opstats"> +<h5><a class="toc-backref" href="#id9">Metrics (OpStats)</a></h5> +<p>A <cite>OpStats</cite> is a set of metrics that represents the statistics of an <cite>operation</cite>. Those metrics +include <cite>success</cite> or <cite>failure</cite> of the operations and its distribution (also known as <cite>Histogram</cite>). +It is usually used for timing.</p> +<pre class="literal-block"> +StatsLogger statsLogger = ...; +OpStatsLogger writeStats = statsLogger.getOpStatsLogger("writes"); +long writeLatency = ...; + +// register success op +writeStats.registerSuccessfulEvent(writeLatency); + +// register failure op +writeStats.registerFailedEvent(writeLatency); +</pre> +</div> +</div> +</div> +<div class="section" id="available-stats-providers"> +<h3><a class="toc-backref" href="#id10">Available Stats Providers</a></h3> +<p>All the available stats providers are listed as below:</p> +<ul class="simple"> +<li>Twitter Science Stats (deprecated)</li> +<li>Twitter Ostrich Stats (deprecated)</li> +<li>Twitter Finagle Stats</li> +<li>Codahale Stats</li> +</ul> +<div class="section" id="twitter-science-stats"> +<h4><a class="toc-backref" href="#id11">Twitter Science Stats</a></h4> +<p>Use following dependency to enable Twitter science stats provider.</p> +<pre class="literal-block"> +<dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>twitter-science-provider</artifactId> + <version>${bookkeeper.version}</version> +</dependency> +</pre> +<p>Construct the stats provider for clients.</p> +<pre class="literal-block"> +StatsProvider statsProvider = new TwitterStatsProvider(); +DistributedLogConfiguration conf = ...; + +// starts the stats provider (optional) +statsProvider.start(conf); + +// all the dl related stats are exposed under "dlog" +StatsLogger statsLogger = statsProvider.getStatsLogger("dlog"); +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .uri(...) + .conf(conf) + .statsLogger(statsLogger) + .build(); + +... + +// stop the stats provider (optional) +statsProvider.stop(); +</pre> +<p>Expose the stats collected by the stats provider by configuring following settings:</p> +<pre class="literal-block"> +// enable exporting the stats +statsExport=true +// exporting the stats at port 8080 +statsHttpPort=8080 +</pre> +<p>If exporting stats is enabled, all the stats are exported by the http endpoint. +You could curl the http endpoint to check the stats.</p> +<pre class="literal-block"> +curl -s <host>:8080/vars +</pre> +<p>check <a class="reference external" href="https://github.com/twitter/commons/tree/master/src/java/com/twitter/common/stats";>ScienceStats</a> for more details.</p> +</div> +<div class="section" id="twitter-ostrich-stats"> +<h4><a class="toc-backref" href="#id12">Twitter Ostrich Stats</a></h4> +<p>Use following dependency to enable Twitter ostrich stats provider.</p> +<pre class="literal-block"> +<dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>twitter-ostrich-provider</artifactId> + <version>${bookkeeper.version}</version> +</dependency> +</pre> +<p>Construct the stats provider for clients.</p> +<pre class="literal-block"> +StatsProvider statsProvider = new TwitterOstrichProvider(); +DistributedLogConfiguration conf = ...; + +// starts the stats provider (optional) +statsProvider.start(conf); + +// all the dl related stats are exposed under "dlog" +StatsLogger statsLogger = statsProvider.getStatsLogger("dlog"); +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .uri(...) + .conf(conf) + .statsLogger(statsLogger) + .build(); + +... + +// stop the stats provider (optional) +statsProvider.stop(); +</pre> +<p>Expose the stats collected by the stats provider by configuring following settings:</p> +<pre class="literal-block"> +// enable exporting the stats +statsExport=true +// exporting the stats at port 8080 +statsHttpPort=8080 +</pre> +<p>If exporting stats is enabled, all the stats are exported by the http endpoint. +You could curl the http endpoint to check the stats.</p> +<pre class="literal-block"> +curl -s <host>:8080/stats.txt +</pre> +<p>check <a class="reference external" href="https://github.com/twitter/ostrich";>Ostrich</a> for more details.</p> +</div> +<div class="section" id="twitter-finagle-metrics"> +<h4><a class="toc-backref" href="#id13">Twitter Finagle Metrics</a></h4> +<p>Use following dependency to enable bridging finagle stats receiver to bookkeeper's stats provider. +All the stats exposed by the stats provider will be collected by finagle stats receiver and exposed +by Twitter's admin service.</p> +<pre class="literal-block"> +<dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>twitter-finagle-provider</artifactId> + <version>${bookkeeper.version}</version> +</dependency> +</pre> +<p>Construct the stats provider for clients.</p> +<pre class="literal-block"> +StatsReceiver statsReceiver = ...; // finagle stats receiver +StatsProvider statsProvider = new FinagleStatsProvider(statsReceiver); +DistributedLogConfiguration conf = ...; + +// the stats provider does nothing on start. +statsProvider.start(conf); + +// all the dl related stats are exposed under "dlog" +StatsLogger statsLogger = statsProvider.getStatsLogger("dlog"); +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .uri(...) + .conf(conf) + .statsLogger(statsLogger) + .build(); + +... + +// the stats provider does nothing on stop. +statsProvider.stop(); +</pre> +<p>check <a class="reference external" href="https://twitter.github.io/twitter-server/Migration.html";>finagle metrics library</a> for more details on how to expose the stats.</p> +</div> +<div class="section" id="codahale-metrics"> +<h4><a class="toc-backref" href="#id14">Codahale Metrics</a></h4> +<p>Use following dependency to enable Twitter ostrich stats provider.</p> +<pre class="literal-block"> +<dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>codahale-metrics-provider</artifactId> + <version>${bookkeeper.version}</version> +</dependency> +</pre> +<p>Construct the stats provider for clients.</p> +<pre class="literal-block"> +StatsProvider statsProvider = new CodahaleMetricsProvider(); +DistributedLogConfiguration conf = ...; + +// starts the stats provider (optional) +statsProvider.start(conf); + +// all the dl related stats are exposed under "dlog" +StatsLogger statsLogger = statsProvider.getStatsLogger("dlog"); +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .uri(...) + .conf(conf) + .statsLogger(statsLogger) + .build(); + +... + +// stop the stats provider (optional) +statsProvider.stop(); +</pre> +<p>Expose the stats collected by the stats provider in different ways by configuring following settings. +Check <a class="reference external" href="https://dropwizard.github.io/metrics/3.1.0/";>Codehale</a> on how to configuring report endpoints.</p> +<pre class="literal-block"> +// How frequent report the stats +codahaleStatsOutputFrequencySeconds=... +// The prefix string of codahale stats +codahaleStatsPrefix=... + +// +// Report Endpoints +// + +// expose the stats to Graphite +codahaleStatsGraphiteEndpoint=... +// expose the stats to CSV files +codahaleStatsCSVEndpoint=... +// expose the stats to Slf4j logging +codahaleStatsSlf4jEndpoint=... +// expose the stats to JMX endpoint +codahaleStatsJmxEndpoint=... +</pre> +<p>check <a class="reference external" href="https://dropwizard.github.io/metrics/3.1.0/";>Codehale</a> for more details.</p> +</div> +</div> +<div class="section" id="enable-stats-provider-on-bookie-servers"> +<h3><a class="toc-backref" href="#id15">Enable Stats Provider on Bookie Servers</a></h3> +<p>The stats provider used by <em>Bookie Servers</em> is configured by setting the following option.</p> +<pre class="literal-block"> +// class of stats provider +statsProviderClass="org.apache.bookkeeper.stats.CodahaleMetricsProvider" +</pre> +</div> +<div class="section" id="metrics"> +<h3><a class="toc-backref" href="#id16">Metrics</a></h3> +<p>Check the <a class="reference external" href="../user_guide/references/metrics">Metrics</a> reference page for the metrics exposed by DistributedLog.</p> +</div> +</div> + + + </div> + </div> +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/49afadb4/docs/0.4.0-incubating/admin_guide/operations.html ---------------------------------------------------------------------- diff --git a/docs/0.4.0-incubating/admin_guide/operations.html b/docs/0.4.0-incubating/admin_guide/operations.html new file mode 100644 index 0000000..8ca4e96 --- /dev/null +++ b/docs/0.4.0-incubating/admin_guide/operations.html @@ -0,0 +1,714 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache DistributedLog (incubating)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/admin_guide/operations.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<div class="row"> + <!-- Sub Navigation --> + <div class="col-sm-3"> + <ul id="sub-nav"> + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html" class="">Admin Guide</a> + + <ul> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html" class="active"> + Operations + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html" class=""> + Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html" class=""> + Performance Tuning + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html" class=""> + Hardware + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html" class=""> + Monitoring + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html" class=""> + ZooKeeper + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html" class=""> + BookKeeper + </a> + + <ul> + + </ul> + + </li> + + </ul> + + </li> + + </ul> + </div> + <!-- Main --> + <div class="col-sm-9"> + <!-- Top anchor --> + <a href="#top"></a> + + <!-- Breadcrumbs above the main heading --> + <ol class="breadcrumb"> + + + + + + + + + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html">Admin Guide</a></li> + + + <li class="active">Operations</li> + </ol> + + <div class="text"> + <!-- Content --> + <div class="contents topic" id="distributedlog-operations"> +<p class="topic-title first">DistributedLog Operations</p> +<ul class="simple"> +<li><a class="reference internal" href="#id1" id="id2">DistributedLog Operations</a><ul> +<li><a class="reference internal" href="#feature-provider" id="id3">Feature Provider</a><ul> +<li><a class="reference internal" href="#api" id="id4">API</a></li> +<li><a class="reference internal" href="#configuration-based-feature-provider" id="id5">Configuration Based Feature Provider</a></li> +<li><a class="reference internal" href="#available-features" id="id6">Available Features</a></li> +</ul> +</li> +<li><a class="reference internal" href="#dlog" id="id7"><cite>dlog</cite></a><ul> +<li><a class="reference internal" href="#create-a-stream" id="id8">Create a stream</a></li> +<li><a class="reference internal" href="#list-the-streams" id="id9">List the streams</a></li> +<li><a class="reference internal" href="#show-stream-s-information" id="id10">Show stream's information</a></li> +<li><a class="reference internal" href="#dump-a-stream" id="id11">Dump a stream</a></li> +<li><a class="reference internal" href="#delete-a-stream" id="id12">Delete a stream</a></li> +<li><a class="reference internal" href="#truncate-a-stream" id="id13">Truncate a stream</a></li> +</ul> +</li> +</ul> +</li> +</ul> +</div> +<div class="section" id="id1"> +<h2><a class="toc-backref" href="#id2">DistributedLog Operations</a></h2> +<div class="section" id="feature-provider"> +<h3><a class="toc-backref" href="#id3">Feature Provider</a></h3> +<p>DistributedLog uses a <cite>feature-provider</cite> library provided by Apache BookKeeper for managing features +dynamically at runtime. It is a <a class="reference external" href="https://en.wikipedia.org/wiki/Feature_toggle";>feature-flag</a> system used to proportionally control what features +are enabled for the system. In other words, it is a way of altering the control in a system without +restarting it. It can be used during all stages of development, its most visible use case is on +production. For instance, during a production release, you can enable or disable individual features, +control the data flow through the system, thereby minimizing risk of system failure in real time.</p> +<p>This <cite>feature-provider</cite> interface is pluggable and easy to integrate with any configuration management +system.</p> +<div class="section" id="api"> +<h4><a class="toc-backref" href="#id4">API</a></h4> +<p><cite>FeatureProvider</cite> is a provider that manages features under different scopes. The provider is responsible +for loading features dynamically at runtime. A <cite>Feature</cite> is a numeric flag that control how much percentage +of this feature will be available to the system - the number is called <cite>availability</cite>.</p> +<pre class="literal-block"> +Feature.name() => returns the name of this feature +Feature.availability() => returns the availability of this feature +Feature.isAvailable() => returns true if its availability is larger than 0; otherwise false +</pre> +<p>It is easy to obtain a feature from the provider by just providing a feature name.</p> +<pre class="literal-block"> +FeatureProvider provider = ...; +Feature feature = provider.getFeature("feature1"); // returns the feature named 'feature1' +</pre> +<p>The <cite>FeatureProvider</cite> is scopable to allow creating features in a hierarchical way. For example, if a system +is comprised of two subsystems, one is <em>cache</em>, while the other one is <em>storage</em>. so the features belong to +different subsystems can be created under different scopes.</p> +<pre class="literal-block"> +FeatureProvider provider = ...; +FeatureProvider cacheFeatureProvider = provider.scope("cache"); +FeatureProvider storageFeatureProvider = provider.scope("storage"); +Feature writeThroughFeature = cacheFeatureProvider.getFeature("write_through"); +Feature duralWriteFeature = storageFeatureProvider.getFeature("dural_write"); + +// so the available features under `provider` are: (assume scopes are separated by '.') +// - 'cache.write_through' +// - 'storage.dural_write' +</pre> +<p>The feature provider could be passed to <cite>DistributedLogNamespaceBuilder</cite> when building the namespace, +thereby it would be used for controlling the features exposed under <cite>DistributedLogNamespace</cite>.</p> +<pre class="literal-block"> +FeatureProvider rootProvider = ...; +FeatureProvider dlFeatureProvider = rootProvider.scope("dlog"); +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .uri(uri) + .conf(conf) + .featureProvider(dlFeatureProvider) + .build(); +</pre> +<p>The feature provider is loaded by reflection on distributedlog write proxy server. You could specify +the feature provider class name as below. Otherwise it would use <cite>DefaultFeatureProvider</cite>, which disables +all the features by default.</p> +<pre class="literal-block"> +featureProviderClass=org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider +</pre> +</div> +<div class="section" id="configuration-based-feature-provider"> +<h4><a class="toc-backref" href="#id5">Configuration Based Feature Provider</a></h4> +<p>Beside <cite>DefaultFeatureProvider</cite>, distributedlog also provides a file-based feature provider - it loads +the features from properties files.</p> +<p>All the features and their availabilities are configured in properties file format. For example,</p> +<pre class="literal-block"> +cache.write_through=100 +storage.dural_write=0 +</pre> +<p>You could configure <cite>featureProviderClass</cite> in distributedlog configuration file by setting it to +<cite>org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider</cite> to enable file-based feature +provider. The feature provider will load the features from two files, one is base config file configured +by <cite>fileFeatureProviderBaseConfigPath</cite>, while the other one is overlay config file configured by +<cite>fileFeatureProviderOverlayConfigPath</cite>. Current implementation doesn't differentiate these two files +too much other than the <cite>overlay</cite> config will override the settings in <cite>base</cite> config. It is recommended +to have a base config file for storing the default availability values for your system and dynamically +adjust the availability values in overlay config file.</p> +<pre class="literal-block"> +featureProviderClass=org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider +fileFeatureProviderBaseConfigPath=/path/to/base/config +fileFeatureProviderOverlayConfigPath=/path/to/overlay/config +// how frequent we reload the config files +dynamicConfigReloadIntervalSec=60 +</pre> +</div> +<div class="section" id="available-features"> +<h4><a class="toc-backref" href="#id6">Available Features</a></h4> +<p>Check the <a class="reference external" href="../user_guide/references/features">Features</a> reference page for the features exposed by DistributedLog.</p> +</div> +</div> +<div class="section" id="dlog"> +<h3><a class="toc-backref" href="#id7"><cite>dlog</cite></a></h3> +<p>A CLI is provided for inspecting DistributedLog streams and metadata.</p> +<pre class="code bash literal-block"> +dlog +JMX enabled by default +Usage: dlog <command> +where <span class="name builtin">command</span> is one of: + <span class="name builtin">local</span> Run distributedlog sandbox + example Run distributedlog example + tool Run distributedlog tool + proxy_tool Run distributedlog proxy tool to interact with proxies + balancer Run distributedlog balancer + admin Run distributedlog admin tool + <span class="name builtin">help</span> This <span class="name builtin">help</span> message + +or <span class="name builtin">command</span> is the full name of a class with a defined main<span class="operator">()</span> method. + +Environment variables: + DLOG_LOG_CONF Log4j configuration file <span class="operator">(</span>default <span class="name variable">$HOME</span>/src/distributedlog/distributedlog-service/conf/log4j.properties<span class="operator">)</span> + DLOG_EXTRA_OPTS Extra options to be passed to the jvm + DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath +</pre> +<p>These variable can also be set in conf/dlogenv.sh</p> +<div class="section" id="create-a-stream"> +<h4><a class="toc-backref" href="#id8">Create a stream</a></h4> +<p>To create a stream:</p> +<pre class="code bash literal-block"> +dlog tool create -u <DL URI> -r <STREAM PREFIX> -e <STREAM EXPRESSION> +</pre> +</div> +<div class="section" id="list-the-streams"> +<h4><a class="toc-backref" href="#id9">List the streams</a></h4> +<p>To list all the streams under a given DistributedLog namespace:</p> +<pre class="code bash literal-block"> +dlog tool list -u <DL URI> +</pre> +</div> +<div class="section" id="show-stream-s-information"> +<h4><a class="toc-backref" href="#id10">Show stream's information</a></h4> +<p>To view the metadata associated with a stream:</p> +<pre class="code bash literal-block"> +dlog tool show -u <DL URI> -s <STREAM NAME> +</pre> +</div> +<div class="section" id="dump-a-stream"> +<h4><a class="toc-backref" href="#id11">Dump a stream</a></h4> +<p>To dump the items inside a stream:</p> +<pre class="code bash literal-block"> +dlog tool dump -u <DL URI> -s <STREAM NAME> -o <START TXN ID> -l <NUM RECORDS> +</pre> +</div> +<div class="section" id="delete-a-stream"> +<h4><a class="toc-backref" href="#id12">Delete a stream</a></h4> +<p>To delete a stream, run:</p> +<pre class="code bash literal-block"> +dlog tool delete -u <DL URI> -s <STREAM NAME> +</pre> +</div> +<div class="section" id="truncate-a-stream"> +<h4><a class="toc-backref" href="#id13">Truncate a stream</a></h4> +<p>Truncate the streams under a given DistributedLog namespace. You could specify a filter to match the streams that you want to truncate.</p> +<p>There is a difference between the <tt class="docutils literal">truncate</tt> and <tt class="docutils literal">delete</tt> command. When you issue a <tt class="docutils literal">truncate</tt>, the data will be purge without removing the streams. A <tt class="docutils literal">delete</tt> will delete the stream. You can pass the flag <tt class="docutils literal"><span class="pre">-delete</span></tt> to the <tt class="docutils literal">truncate</tt> command to also delete the streams.</p> +<pre class="code bash literal-block"> +dlog tool truncate -u <DL URI> +</pre> +</div> +</div> +</div> + + + </div> + </div> +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/49afadb4/docs/0.4.0-incubating/admin_guide/performance.html ---------------------------------------------------------------------- diff --git a/docs/0.4.0-incubating/admin_guide/performance.html b/docs/0.4.0-incubating/admin_guide/performance.html new file mode 100644 index 0000000..f940f91 --- /dev/null +++ b/docs/0.4.0-incubating/admin_guide/performance.html @@ -0,0 +1,543 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache DistributedLog (incubating)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/admin_guide/performance.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<div class="row"> + <!-- Sub Navigation --> + <div class="col-sm-3"> + <ul id="sub-nav"> + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html" class="">Admin Guide</a> + + <ul> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html" class=""> + Operations + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html" class=""> + Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html" class=""> + Global Cluster Setup + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html" class="active"> + Performance Tuning + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html" class=""> + Load Test + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html" class=""> + Hardware + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html" class=""> + Monitoring + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html" class=""> + ZooKeeper + </a> + + <ul> + + </ul> + + </li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html" class=""> + BookKeeper + </a> + + <ul> + + </ul> + + </li> + + </ul> + + </li> + + </ul> + </div> + <!-- Main --> + <div class="col-sm-9"> + <!-- Top anchor --> + <a href="#top"></a> + + <!-- Breadcrumbs above the main heading --> + <ol class="breadcrumb"> + + + + + + + + + + + + + <li><a href="/docs/0.4.0-incubating/admin_guide/main.html">Admin Guide</a></li> + + + <li class="active">Performance Tuning</li> + </ol> + + <div class="text"> + <!-- Content --> + <div class="contents topic" id="performance-tuning"> +<p class="topic-title first">Performance Tuning</p> +<ul class="simple"> +<li><a class="reference internal" href="#id1" id="id2">Performance Tuning</a></li> +</ul> +</div> +<div class="section" id="id1"> +<h2><a class="toc-backref" href="#id2">Performance Tuning</a></h2> +<p>(describe how to tune performance, critical settings)</p> +</div> + + + </div> + </div> +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/49afadb4/docs/0.4.0-incubating/admin_guide/vagrant.rst ---------------------------------------------------------------------- diff --git a/docs/0.4.0-incubating/admin_guide/vagrant.rst b/docs/0.4.0-incubating/admin_guide/vagrant.rst new file mode 100644 index 0000000..e4ed574 --- /dev/null +++ b/docs/0.4.0-incubating/admin_guide/vagrant.rst @@ -0,0 +1,18 @@ +Sample Deployment using vagrant +================================ + +This file explains vagrant deployment. + +Prerequesites +-------------- +1. Vagrant: From https://www.vagrantup.com/downloads.html +2. vagrant-hostmanager plugin: From https://github.com/devopsgroup-io/vagrant-hostmanager + +Steps +----- +1. Create a snapshot using ./scripts/snapshot +2. Run vagrant up from the root directory of the enlistment +3. Vagrant brings up a zookeepers with machine names: zk1,zk2, .... with IP addresses 192.168.50.11,192.168.50.12,.... +4. Vagrant brings the bookies with machine names: bk1,bk2, ... with IP addresses 192.168.50.51,192.168.50.52,.... +5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script. +6. If you want to run the client on the host machine, please add these node names and their IP addresses to the /etc/hosts file on the host.
