http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/tutorials/analytics-mapreduce.html ---------------------------------------------------------------------- diff --git a/content/docs/0.4.0-incubating/tutorials/analytics-mapreduce.html b/content/docs/0.4.0-incubating/tutorials/analytics-mapreduce.html new file mode 100644 index 0000000..a737927 --- /dev/null +++ b/content/docs/0.4.0-incubating/tutorials/analytics-mapreduce.html @@ -0,0 +1,569 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Tutorial - DistributedLog meets MapReduce</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/tutorials/analytics-mapreduce.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<div class="col-md-8 col-md-offset-2"> + <div class="contents topic" id="tutorial-distributedlog-meets-mapreduce"> +<p class="topic-title first">Tutorial - DistributedLog meets MapReduce</p> +<ul class="simple"> +<li><a class="reference internal" href="#distributedlog-meets-mapreduce" id="id1">DistributedLog meets MapReduce</a><ul> +<li><a class="reference internal" href="#inputformat" id="id2">InputFormat</a></li> +<li><a class="reference internal" href="#log-segment-vs-data-split" id="id3">Log Segment vs Data Split</a></li> +<li><a class="reference internal" href="#log-segment-record-reader" id="id4">Log Segment Record Reader</a></li> +</ul> +</li> +</ul> +</div> +<div class="section" id="distributedlog-meets-mapreduce"> +<h2><a class="toc-backref" href="#id1">DistributedLog meets MapReduce</a></h2> +<p>A distributedlog log stream is consists of log segments. Each log segment is distributed +among multiple bookies node. This nature of data distribution allows distributedlog easily +integrated with any analytics processing systems like <em>MapReduce</em> and <em>Spark</em>. This tutorial +shows how you could use <em>MapReduce</em> to process log streams' data in batch and how <em>MapReduce</em> +can leverage the data locality of log segments.</p> +<div class="section" id="inputformat"> +<h3><a class="toc-backref" href="#id2">InputFormat</a></h3> +<p><strong>InputFormat</strong> is one of the fundamental class in Hadoop MapReduce framework, that is used +for accessing data from different sources. The class is responsible for defining two main +things:</p> +<ul class="simple"> +<li>Data Splits</li> +<li>Record Reader</li> +</ul> +<p><em>Data Split</em> is a fundamental concept in Hadoop MapReduce framework which defines both +the size of individual Map tasks and its potential execution server. The <em>Record Reader</em> is +responsible for actual reading records from the <em>data split</em> and submitting them (as key/value +pairs) to the mapper.</p> +<p>Using distributedlog log streams as the sources for a MapReduce job, the <em>log segments</em> are +the <em>data splits</em>, while the <em>log segment reader</em> for a log segment is the <em>record reader</em> for +a <em>data split</em>.</p> +</div> +<div class="section" id="log-segment-vs-data-split"> +<h3><a class="toc-backref" href="#id3">Log Segment vs Data Split</a></h3> +<p>Any split implementation extends the Apache base abstract class - <strong>InputSplit</strong>, defining a +split length and locations. A distributedlog log segment has <em>record count</em>, which could be used +to define the length of the split, and its metadata contains the storage nodes that are used to +store its log records, which could be used to define the locations of the split. So we could +create a <strong>LogSegmentSplit</strong> wrapping over a <em>LogSegment</em> (LogSegmentMetadata and LedgerMetadata).</p> +<pre class="literal-block"> +public class LogSegmentSplit extends InputSplit { + + private LogSegmentMetadata logSegmentMetadata; + private LedgerMetadata ledgerMetadata; + + public LogSegmentSplit() {} + + public LogSegmentSplit(LogSegmentMetadata logSegmentMetadata, + LedgerMetadata ledgerMetadata) { + this.logSegmentMetadata = logSegmentMetadata; + this.ledgerMetadata = ledgerMetadata; + } + +} +</pre> +<p>The length of the log segment split is the <em>number of records in the log segment</em>.</p> +<pre class="literal-block"> +@Override +public long getLength() + throws IOException, InterruptedException { + return logSegmentMetadata.getRecordCount(); +} +</pre> +<p>The locations of the log segment split are the bookies' addresses in the ensembles of +the log segment.</p> +<pre class="literal-block"> +@Override +public String[] getLocations() + throws IOException, InterruptedException { + Set<String> locations = Sets.newHashSet(); + for (ArrayList<BookieSocketAddress> ensemble : ledgerMetadata.getEnsembles().values()) { + for (BookieSocketAddress host : ensemble) { + locations.add(host.getHostName()); + } + } + return locations.toArray(new String[locations.size()]); +} +</pre> +<p>At this point, we will have a basic <strong>LogSegmentSplit</strong> wrapping <em>LogSegmentMetadata</em> and +<em>LedgerMetadata</em>. Then we could retrieve the list of log segments of a log stream and construct +corresponding <em>data splits</em> in distributedlog inputformat.</p> +<pre class="literal-block"> +public class DistributedLogInputFormat + extends InputFormat<DLSN, LogRecordWithDLSN> implements Configurable { + + @Override + public List<InputSplit> getSplits(JobContext jobContext) + throws IOException, InterruptedException { + List<LogSegmentMetadata> segments = dlm.getLogSegments(); + List<InputSplit> inputSplits = Lists.newArrayListWithCapacity(segments.size()); + BookKeeper bk = namespace.getReaderBKC().get(); + LedgerManager lm = BookKeeperAccessor.getLedgerManager(bk); + final AtomicInteger rcHolder = new AtomicInteger(0); + final AtomicReference<LedgerMetadata> metadataHolder = new AtomicReference<LedgerMetadata>(null); + for (LogSegmentMetadata segment : segments) { + final CountDownLatch latch = new CountDownLatch(1); + lm.readLedgerMetadata(segment.getLedgerId(), + new BookkeeperInternalCallbacks.GenericCallback<LedgerMetadata>() { + @Override + public void operationComplete(int rc, LedgerMetadata ledgerMetadata) { + metadataHolder.set(ledgerMetadata); + rcHolder.set(rc); + latch.countDown(); + } + }); + latch.await(); + if (BKException.Code.OK != rcHolder.get()) { + throw new IOException("Faild to get log segment metadata for " + segment + " : " + + BKException.getMessage(rcHolder.get())); + } + inputSplits.add(new LogSegmentSplit(segment, metadataHolder.get())); + } + return inputSplits; + } + +} +</pre> +</div> +<div class="section" id="log-segment-record-reader"> +<h3><a class="toc-backref" href="#id4">Log Segment Record Reader</a></h3> +<p>At this point, we know how to break the log streams into <em>data splits</em>. Then we need to be able +to create a <strong>RecordReader</strong> for individual <em>data split</em>. Since each <em>data split</em> is effectively +a <em>log segment</em> in distributedlog, it is straight to implement it using distributedlog's log segment +reader. For simplicity, this example uses the raw bk api to access entries, which it doesn't +leverage features like <strong>ReadAhead</strong> provided in distributedlog. It could be changed to +use log segment reader for better performance.</p> +<p>From the <em>data split</em>, we know which log segment and its corresponding bookkeeper ledger. Then +we could open the ledger handle when initializing the record reader.</p> +<pre class="literal-block"> +LogSegmentReader(String streamName, + DistributedLogConfiguration conf, + BookKeeper bk, + LogSegmentSplit split) + throws IOException { + this.streamName = streamName; + this.bk = bk; + this.metadata = split.getMetadata(); + try { + this.lh = bk.openLedgerNoRecovery( + split.getLedgerId(), + BookKeeper.DigestType.CRC32, + conf.getBKDigestPW().getBytes(UTF_8)); + } catch (BKException e) { + throw new IOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } +} +</pre> +<p>Reading records from the <em>data split</em> is effectively reading records from the distributedlog +log segment.</p> +<pre class="literal-block"> +try { + Enumeration<LedgerEntry> entries = + lh.readEntries(entryId, entryId); + if (entries.hasMoreElements()) { + LedgerEntry entry = entries.nextElement(); + Entry.newBuilder() + .setLogSegmentInfo(metadata.getLogSegmentSequenceNumber(), + metadata.getStartSequenceId()) + .setEntryId(entry.getEntryId()) + .setEnvelopeEntry( + LogSegmentMetadata.supportsEnvelopedEntries(metadata.getVersion())) + .deserializeRecordSet(true) + .setInputStream(entry.getEntryInputStream()) + .buildReader(); + } + return nextKeyValue(); +} catch (BKException e) { + throw new IOException(e); +} +</pre> +<p>We could calculate the progress by comparing the position with the record count of this log segment.</p> +<pre class="literal-block"> +@Override +public float getProgress() + throws IOException, InterruptedException { + if (metadata.getRecordCount() > 0) { + return ((float) (readPos + 1)) / metadata.getRecordCount(); + } + return 1; +} +</pre> +<p>Once we have <em>LogSegmentSplit</em> and the <em>LogSegmentReader</em> over a split. We could hook them up to +implement distributedlog's InputFormat. Please check out the <a class="reference external" href="https://github.com/apache/incubator-distributedlog/tree/master/distributedlog-tutorials/distributedlog-mapreduce";>code</a> for more details.</p> +</div> +</div> + + +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/tutorials/basic-1.html ---------------------------------------------------------------------- diff --git a/content/docs/0.4.0-incubating/tutorials/basic-1.html b/content/docs/0.4.0-incubating/tutorials/basic-1.html new file mode 100644 index 0000000..c8b9d06 --- /dev/null +++ b/content/docs/0.4.0-incubating/tutorials/basic-1.html @@ -0,0 +1,612 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>API - Write Records (via core library)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/tutorials/basic-1.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<div class="col-md-8 col-md-offset-2"> + <div class="contents topic" id="basic-tutorial-using-core-library-to-write-records"> +<p class="topic-title first">Basic Tutorial - Using Core Library to write records</p> +<ul class="auto-toc simple"> +<li><a class="reference internal" href="#basic-tutorial-write-records-using-core-library" id="id2">1 Basic Tutorial - Write Records using Core Library</a><ul class="auto-toc"> +<li><a class="reference internal" href="#open-a-writer" id="id3">1.1 Open a writer</a><ul class="auto-toc"> +<li><a class="reference internal" href="#create-distributedlog-uri" id="id4">1.1.1 Create distributedlog URI</a></li> +<li><a class="reference internal" href="#create-distributedlog-configuration" id="id5">1.1.2 Create distributedlog configuration</a></li> +<li><a class="reference internal" href="#enable-immediate-flush" id="id6">1.1.3 Enable immediate flush</a></li> +<li><a class="reference internal" href="#enable-immediate-locking" id="id7">1.1.4 Enable immediate locking</a></li> +<li><a class="reference internal" href="#build-the-distributedlog-namespace" id="id8">1.1.5 Build the distributedlog namespace</a></li> +<li><a class="reference internal" href="#open-the-writer" id="id9">1.1.6 Open the writer</a></li> +</ul> +</li> +<li><a class="reference internal" href="#write-records" id="id10">1.2 Write Records</a><ul class="auto-toc"> +<li><a class="reference internal" href="#construct-a-log-record" id="id11">1.2.1 Construct a log record</a></li> +<li><a class="reference internal" href="#write-the-log-record" id="id12">1.2.2 Write the log record</a></li> +<li><a class="reference internal" href="#register-the-write-callback" id="id13">1.2.3 Register the write callback</a></li> +</ul> +</li> +<li><a class="reference internal" href="#close-the-writer" id="id14">1.3 Close the writer</a><ul class="auto-toc"> +<li><a class="reference internal" href="#close-the-writer-after-usage" id="id15">1.3.1 Close the writer after usage</a></li> +</ul> +</li> +<li><a class="reference internal" href="#run-the-tutorial" id="id16">1.4 Run the tutorial</a><ul class="auto-toc"> +<li><a class="reference internal" href="#start-the-local-bookkeeper-cluster" id="id17">1.4.1 Start the local bookkeeper cluster</a></li> +<li><a class="reference internal" href="#create-the-stream" id="id18">1.4.2 Create the stream</a></li> +<li><a class="reference internal" href="#tail-the-stream" id="id19">1.4.3 Tail the stream</a></li> +<li><a class="reference internal" href="#id1" id="id20">1.4.4 Write records</a></li> +<li><a class="reference internal" href="#check-the-results" id="id21">1.4.5 Check the results</a></li> +<li><a class="reference internal" href="#attempt-a-second-writer" id="id22">1.4.6 Attempt a second writer</a></li> +</ul> +</li> +</ul> +</li> +</ul> +</div> +<div class="section" id="basic-tutorial-write-records-using-core-library"> +<h2><a class="toc-backref" href="#id2">1 Basic Tutorial - Write Records using Core Library</a></h2> +<p>This tutorial shows how to write records using core library.</p> +<div class="section" id="open-a-writer"> +<h3><a class="toc-backref" href="#id3">1.1 Open a writer</a></h3> +<p>Before everything, you have to open a writer to write records. +These are the steps to follow to <cite>open a writer</cite>.</p> +<div class="section" id="create-distributedlog-uri"> +<h4><a class="toc-backref" href="#id4">1.1.1 Create distributedlog URI</a></h4> +<pre class="literal-block"> +String dlUriStr = ...; +URI uri = URI.create(dlUriStr); +</pre> +</div> +<div class="section" id="create-distributedlog-configuration"> +<h4><a class="toc-backref" href="#id5">1.1.2 Create distributedlog configuration</a></h4> +<pre class="literal-block"> +DistributedLogConfiguration conf = new DistributedLogConfiguration(); +</pre> +</div> +<div class="section" id="enable-immediate-flush"> +<h4><a class="toc-backref" href="#id6">1.1.3 Enable immediate flush</a></h4> +<pre class="literal-block"> +conf.setImmediateFlushEnabled(true); +conf.setOutputBufferSize(0); +conf.setPeriodicFlushFrequencyMilliSeconds(0); +</pre> +</div> +<div class="section" id="enable-immediate-locking"> +<h4><a class="toc-backref" href="#id7">1.1.4 Enable immediate locking</a></h4> +<p>So if there is already a writer wring to the stream, opening another writer will +fail because previous writer already held a lock.</p> +<pre class="literal-block"> +conf.setLockTimeout(DistributedLogConstants.LOCK_IMMEDIATE); +</pre> +</div> +<div class="section" id="build-the-distributedlog-namespace"> +<h4><a class="toc-backref" href="#id8">1.1.5 Build the distributedlog namespace</a></h4> +<pre class="literal-block"> +DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() + .conf(conf) + .uri(uri) + .regionId(DistributedLogConstants.LOCAL_REGION_ID) + .clientId("console-writer") + .build(); +</pre> +</div> +<div class="section" id="open-the-writer"> +<h4><a class="toc-backref" href="#id9">1.1.6 Open the writer</a></h4> +<pre class="literal-block"> +DistributedLogManager dlm = namespace.openLog("basic-stream-1"); +AsyncLogWriter writer = FutureUtils.result(dlm.openAsyncLogWriter()); +</pre> +</div> +</div> +<div class="section" id="write-records"> +<h3><a class="toc-backref" href="#id10">1.2 Write Records</a></h3> +<p>Once you got a <cite>writer</cite> instance, you can start writing <cite>records</cite> into the stream.</p> +<div class="section" id="construct-a-log-record"> +<h4><a class="toc-backref" href="#id11">1.2.1 Construct a log record</a></h4> +<p>Here lets use <cite>System.currentTimeMillis()</cite> as the <cite>TransactionID</cite>.</p> +<pre class="literal-block"> +byte[] data = ...; +LogRecord record = new LogRecord(System.currentTimeMillis(), data); +</pre> +</div> +<div class="section" id="write-the-log-record"> +<h4><a class="toc-backref" href="#id12">1.2.2 Write the log record</a></h4> +<pre class="literal-block"> +Future<DLSN> writeFuture = writer.write(record); +</pre> +</div> +<div class="section" id="register-the-write-callback"> +<h4><a class="toc-backref" href="#id13">1.2.3 Register the write callback</a></h4> +<p>Register a future listener on write completion. The writer will be notified once the write is completed.</p> +<pre class="literal-block"> +writeFuture.addEventListener(new FutureEventListener<DLSN>() { + @Override + public void onFailure(Throwable cause) { + // executed when write failed. + } + + @Override + public void onSuccess(DLSN value) { + // executed when write completed. + } +}); +</pre> +</div> +</div> +<div class="section" id="close-the-writer"> +<h3><a class="toc-backref" href="#id14">1.3 Close the writer</a></h3> +<div class="section" id="close-the-writer-after-usage"> +<h4><a class="toc-backref" href="#id15">1.3.1 Close the writer after usage</a></h4> +<pre class="literal-block"> +FutureUtils.result(writer.asyncClose()); +</pre> +</div> +</div> +<div class="section" id="run-the-tutorial"> +<h3><a class="toc-backref" href="#id16">1.4 Run the tutorial</a></h3> +<p>Run the example in the following steps:</p> +<div class="section" id="start-the-local-bookkeeper-cluster"> +<h4><a class="toc-backref" href="#id17">1.4.1 Start the local bookkeeper cluster</a></h4> +<p>You can use follow command to start the distributedlog stack locally. +After the distributedlog is started, you could access it using +distributedlog uri <em>distributedlog://127.0.0.1:7000/messaging/distributedlog</em>.</p> +<pre class="literal-block"> +// dlog local ${zk-port} +./distributedlog-core/bin/dlog local 7000 +</pre> +</div> +<div class="section" id="create-the-stream"> +<h4><a class="toc-backref" href="#id18">1.4.2 Create the stream</a></h4> +<pre class="literal-block"> +// Create Stream `basic-stream-1` +// dlog tool create -u ${distributedlog-uri} -r ${stream-prefix} -e ${stream-regex} +./distributedlog-core/bin/dlog tool create -u distributedlog://127.0.0.1:7000/messaging/distributedlog -r basic-stream- -e 1 +</pre> +</div> +<div class="section" id="tail-the-stream"> +<h4><a class="toc-backref" href="#id19">1.4.3 Tail the stream</a></h4> +<p>Tailing the stream using <cite>TailReader</cite> to wait for new records.</p> +<pre class="literal-block"> +// Tailing Stream `basic-stream-1` +// runner run org.apache.distributedlog.basic.TailReader ${distributedlog-uri} ${stream} +./distributedlog-tutorials/distributedlog-basic/bin/runner run org.apache.distributedlog.basic.TailReader distributedlog://127.0.0.1:7000/messaging/distributedlog basic-stream-1 +</pre> +</div> +<div class="section" id="id1"> +<h4><a class="toc-backref" href="#id20">1.4.4 Write records</a></h4> +<p>Run the example to write records to the stream in a console.</p> +<pre class="literal-block"> +// Write Records into Stream `basic-stream-1` +// runner run org.apache.distributedlog.basic.ConsoleWriter ${distributedlog-uri} ${stream} +./distributedlog-tutorials/distributedlog-basic/bin/runner run org.apache.distributedlog.basic.ConsoleWriter distributedlog://127.0.0.1:7000/messaging/distributedlog basic-stream-1 +</pre> +</div> +<div class="section" id="check-the-results"> +<h4><a class="toc-backref" href="#id21">1.4.5 Check the results</a></h4> +<p>Example output from <cite>ConsoleWriter</cite> and <cite>TailReader</cite>.</p> +<pre class="literal-block"> +// Output of `ConsoleWriter` +Opening log stream basic-stream-1 +[dlog] > test! +[dlog] > + + +// Output of `TailReader` +Opening log stream basic-stream-1 +Log stream basic-stream-1 is empty. +Wait for records starting from DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0} +Received record DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0} +""" +test! +""" +</pre> +</div> +<div class="section" id="attempt-a-second-writer"> +<h4><a class="toc-backref" href="#id22">1.4.6 Attempt a second writer</a></h4> +<p>Open another terminal to run <cite>ConsoleWriter</cite>. It would fail with <cite>OwnershipAcquireFailedException</cite> as previous +<cite>ConsoleWriter</cite> is still holding lock on writing to stream <cite>basic-stream-1</cite>.</p> +<pre class="literal-block"> +Opening log stream basic-stream-1 +Exception in thread "main" org.apache.distributedlog.exceptions.OwnershipAcquireFailedException: LockPath - /messaging/distributedlog/basic-stream-1/<default>/lock: Lock acquisition failed, the current owner is console-writer + at org.apache.distributedlog.lock.ZKSessionLock$8.apply(ZKSessionLock.java:570) + at org.apache.distributedlog.lock.ZKSessionLock$8.apply(ZKSessionLock.java:567) + at com.twitter.util.Future$$anonfun$map$1$$anonfun$apply$8.apply(Future.scala:1041) + at com.twitter.util.Try$.apply(Try.scala:13) + at com.twitter.util.Future$.apply(Future.scala:132) + at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:1041) + at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:1040) + at com.twitter.util.Promise$Transformer.liftedTree1$1(Promise.scala:112) + at com.twitter.util.Promise$Transformer.k(Promise.scala:112) + at com.twitter.util.Promise$Transformer.apply(Promise.scala:122) + at com.twitter.util.Promise$Transformer.apply(Promise.scala:103) + at com.twitter.util.Promise$$anon$1.run(Promise.scala:357) + at com.twitter.concurrent.LocalScheduler$Activation.run(Scheduler.scala:178) + at com.twitter.concurrent.LocalScheduler$Activation.submit(Scheduler.scala:136) + at com.twitter.concurrent.LocalScheduler.submit(Scheduler.scala:207) + at com.twitter.concurrent.Scheduler$.submit(Scheduler.scala:92) + at com.twitter.util.Promise.runq(Promise.scala:350) + at com.twitter.util.Promise.updateIfEmpty(Promise.scala:716) + at com.twitter.util.Promise.update(Promise.scala:694) + at com.twitter.util.Promise.setValue(Promise.scala:670) + at org.apache.distributedlog.lock.ZKSessionLock$9.safeRun(ZKSessionLock.java:622) + at org.apache.bookkeeper.util.SafeRunnable.run(SafeRunnable.java:31) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) + at java.util.concurrent.FutureTask.run(FutureTask.java:262) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) +</pre> +</div> +</div> +</div> + + +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/tutorials/basic-2.html ---------------------------------------------------------------------- diff --git a/content/docs/0.4.0-incubating/tutorials/basic-2.html b/content/docs/0.4.0-incubating/tutorials/basic-2.html new file mode 100644 index 0000000..88c464e --- /dev/null +++ b/content/docs/0.4.0-incubating/tutorials/basic-2.html @@ -0,0 +1,580 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>API - Write Records (via write proxy)</title> + <meta name="description" content="Apache DistributedLog is an high performance replicated log. +"> + + <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css"> + <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css"> + <!-- JQuery --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/tutorials/basic-2.html"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml";> + <!-- Font Awesome --> + <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script> + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-83870961-1', 'auto'); + ga('send', 'pageview'); + </script> + <!-- End Google Analytics --> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <!-- Overview --> + <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li> + <!-- Concepts --> + <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li> + <!-- Quick Start --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + + + <li> + <a href="/docs/0.4.0-incubating/start/building.html"> + Build DistributedLog from Source + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/start/download.html"> + Download Releases + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Quickstart</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/start/quickstart.html"> + Setup & Run Example + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-1.html"> + API - Write Records (via core library) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-2.html"> + API - Write Records (via write proxy) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/basic-5.html"> + API - Read Records + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Deployment</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/deployment/cluster.html"> + Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/global-cluster.html"> + Global Cluster Setup + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/deployment/docker.html"> + Docker + </a> + </li> + + </ul> + </li> + <!-- API --> + <li> + <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a> + <ul class="dropdown-menu" role="menu"> + <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li> + </ul> + </li> + <!-- User Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + + + <li> + <a href="/docs/0.4.0-incubating/basics/introduction.html"> + Introduction + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html"> + Considerations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html"> + Architecture + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/api/main.html"> + API + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html"> + Configuration + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/design/main.html"> + Detail Design + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html"> + Global Replicated Log + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html"> + Implementation + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/user_guide/references/main.html"> + References + </a> + </li> + + </ul> + </li> + <!-- Admin Guide --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li> + + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/operations.html"> + Operations + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/performance.html"> + Performance Tuning + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html"> + Load Test + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/hardware.html"> + Hardware + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html"> + Monitoring + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html"> + ZooKeeper + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html"> + BookKeeper + </a> + </li> + + </ul> + </li> + <!-- Tutorials --> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header"><strong>Basic</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li> + <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Messaging</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html"> + Write records to partitioned streams + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html"> + Write records to multiple streams (load balancer) + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html"> + At-least-once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html"> + Exact-Once Processing + </a> + </li> + + <li> + <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html"> + Implement a kafka-like pub/sub system + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Replicated State Machines</strong></li> + + + <li> + <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html"> + Build replicated state machines + </a> + </li> + + <li role="separator" class="divider"></li> + <li class="dropdown-header"><strong>Analytics</strong></li> + <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + + <!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<div class="col-md-8 col-md-offset-2"> + <div class="contents topic" id="basic-tutorial-using-proxy-client-to-write-records"> +<p class="topic-title first">Basic Tutorial - Using Proxy Client to write records</p> +<ul class="auto-toc simple"> +<li><a class="reference internal" href="#basic-tutorial-write-records-using-write-proxy-client" id="id2">1 Basic Tutorial - Write Records using Write Proxy Client</a><ul class="auto-toc"> +<li><a class="reference internal" href="#open-a-write-proxy-client" id="id3">1.1 Open a write proxy client</a><ul class="auto-toc"> +<li><a class="reference internal" href="#create-write-proxy-client-builder" id="id4">1.1.1 Create write proxy client builder</a></li> +<li><a class="reference internal" href="#enable-thrift-mux" id="id5">1.1.2 Enable thrift mux</a></li> +<li><a class="reference internal" href="#point-the-client-to-write-proxy-using-finagle-name" id="id6">1.1.3 Point the client to write proxy using finagle name</a></li> +<li><a class="reference internal" href="#build-the-write-proxy-client" id="id7">1.1.4 Build the write proxy client</a></li> +</ul> +</li> +<li><a class="reference internal" href="#write-records" id="id8">1.2 Write Records</a><ul class="auto-toc"> +<li><a class="reference internal" href="#write-records-to-a-stream" id="id9">1.2.1 Write records to a stream</a></li> +<li><a class="reference internal" href="#register-the-write-callback" id="id10">1.2.2 Register the write callback</a></li> +</ul> +</li> +<li><a class="reference internal" href="#close-the-write-proxy-client" id="id11">1.3 Close the write proxy client</a></li> +<li><a class="reference internal" href="#run-the-tutorial" id="id12">1.4 Run the tutorial</a><ul class="auto-toc"> +<li><a class="reference internal" href="#start-the-local-bookkeeper-cluster" id="id13">1.4.1 Start the local bookkeeper cluster</a></li> +<li><a class="reference internal" href="#start-the-write-proxy" id="id14">1.4.2 Start the write proxy</a></li> +<li><a class="reference internal" href="#create-the-stream" id="id15">1.4.3 Create the stream</a></li> +<li><a class="reference internal" href="#tail-the-stream" id="id16">1.4.4 Tail the stream</a></li> +<li><a class="reference internal" href="#id1" id="id17">1.4.5 Write records</a></li> +<li><a class="reference internal" href="#check-the-results" id="id18">1.4.6 Check the results</a></li> +<li><a class="reference internal" href="#attempt-a-second-writer" id="id19">1.4.7 Attempt a second writer</a></li> +</ul> +</li> +</ul> +</li> +</ul> +</div> +<div class="section" id="basic-tutorial-write-records-using-write-proxy-client"> +<h2><a class="toc-backref" href="#id2">1 Basic Tutorial - Write Records using Write Proxy Client</a></h2> +<p>This tutorial shows how to write records using write proxy client.</p> +<div class="section" id="open-a-write-proxy-client"> +<h3><a class="toc-backref" href="#id3">1.1 Open a write proxy client</a></h3> +<div class="section" id="create-write-proxy-client-builder"> +<h4><a class="toc-backref" href="#id4">1.1.1 Create write proxy client builder</a></h4> +<pre class="literal-block"> +DistributedLogClientBuilder builder = DistributedLogClientBuilder.newBuilder(); + .clientId(ClientId.apply("console-proxy-writer")) + .name("console-proxy-writer"); +</pre> +</div> +<div class="section" id="enable-thrift-mux"> +<h4><a class="toc-backref" href="#id5">1.1.2 Enable thrift mux</a></h4> +<pre class="literal-block"> +builder = builder.thriftmux(true); +</pre> +</div> +<div class="section" id="point-the-client-to-write-proxy-using-finagle-name"> +<h4><a class="toc-backref" href="#id6">1.1.3 Point the client to write proxy using finagle name</a></h4> +<pre class="literal-block"> +String finagleNameStr = "inet!127.0.0.1:8000"; +builder = builder.finagleNameStr(finagleNameStr); +</pre> +</div> +<div class="section" id="build-the-write-proxy-client"> +<h4><a class="toc-backref" href="#id7">1.1.4 Build the write proxy client</a></h4> +<pre class="literal-block"> +DistributedLogClient client = builder.build(); +</pre> +</div> +</div> +<div class="section" id="write-records"> +<h3><a class="toc-backref" href="#id8">1.2 Write Records</a></h3> +<div class="section" id="write-records-to-a-stream"> +<h4><a class="toc-backref" href="#id9">1.2.1 Write records to a stream</a></h4> +<p>Application does not have to provide <cite>TransactionID</cite> on writing. +The <cite>TransactionID</cite> of a record is assigned by the write proxy.</p> +<pre class="literal-block"> +String streamName = "basic-stream-2"; +byte[] data = ...; +Future<DLSN> writeFuture = client.write(streamName, ByteBuffer.wrap(data)); +</pre> +</div> +<div class="section" id="register-the-write-callback"> +<h4><a class="toc-backref" href="#id10">1.2.2 Register the write callback</a></h4> +<p>Register a future listener on write completion. The writer will be notified once the write is completed.</p> +<pre class="literal-block"> +writeFuture.addEventListener(new FutureEventListener<DLSN>() { + @Override + public void onFailure(Throwable cause) { + // executed when write failed. + } + + @Override + public void onSuccess(DLSN value) { + // executed when write completed. + } +}); +</pre> +</div> +</div> +<div class="section" id="close-the-write-proxy-client"> +<h3><a class="toc-backref" href="#id11">1.3 Close the write proxy client</a></h3> +<p>Close the write proxy client after usage.</p> +<pre class="literal-block"> +client.close(); +</pre> +</div> +<div class="section" id="run-the-tutorial"> +<h3><a class="toc-backref" href="#id12">1.4 Run the tutorial</a></h3> +<p>Run the example in the following steps:</p> +<div class="section" id="start-the-local-bookkeeper-cluster"> +<h4><a class="toc-backref" href="#id13">1.4.1 Start the local bookkeeper cluster</a></h4> +<p>You can use follow command to start the distributedlog stack locally. +After the distributedlog cluster is started, you could access it using +distributedlog uri <em>distributedlog://127.0.0.1:7000/messaging/distributedlog</em>.</p> +<pre class="literal-block"> +// dlog local ${zk-port} +./distributedlog-core/bin/dlog local 7000 +</pre> +</div> +<div class="section" id="start-the-write-proxy"> +<h4><a class="toc-backref" href="#id14">1.4.2 Start the write proxy</a></h4> +<p>Start the write proxy, listening on port 8000.</p> +<pre class="literal-block"> +// DistributedLogServerApp -p ${service-port} --shard-id ${shard-id} -sp ${stats-port} -u {distributedlog-uri} -mx -c ${conf-file} +./distributedlog-service/bin/dlog org.apache.distributedlog.service.DistributedLogServerApp -p 8000 --shard-id 1 -sp 8001 -u distributedlog://127.0.0.1:7000/messaging/distributedlog -mx -c ${distributedlog-repo}/distributedlog-service/conf/distributedlog_proxy.conf +</pre> +</div> +<div class="section" id="create-the-stream"> +<h4><a class="toc-backref" href="#id15">1.4.3 Create the stream</a></h4> +<p>Create the stream under the distributedlog uri.</p> +<pre class="literal-block"> +// Create Stream `basic-stream-2` +// dlog tool create -u ${distributedlog-uri} -r ${stream-prefix} -e ${stream-regex} +./distributedlog-core/bin/dlog tool create -u distributedlog://127.0.0.1:7000/messaging/distributedlog -r basic-stream- -e 2 +</pre> +</div> +<div class="section" id="tail-the-stream"> +<h4><a class="toc-backref" href="#id16">1.4.4 Tail the stream</a></h4> +<p>Tailing the stream using <cite>TailReader</cite> to wait for new records.</p> +<pre class="literal-block"> +// Tailing Stream `basic-stream-2` +// runner run org.apache.distributedlog.basic.TailReader ${distributedlog-uri} ${stream} +./distributedlog-tutorials/distributedlog-basic/bin/runner run org.apache.distributedlog.basic.TailReader distributedlog://127.0.0.1:7000/messaging/distributedlog basic-stream-2 +</pre> +</div> +<div class="section" id="id1"> +<h4><a class="toc-backref" href="#id17">1.4.5 Write records</a></h4> +<p>Run the example to write records to the stream in a console.</p> +<pre class="literal-block"> +// Write Records into Stream `basic-stream-2` +// runner run org.apache.distributedlog.basic.ConsoleProxyWriter ${distributedlog-uri} ${stream} +./distributedlog-tutorials/distributedlog-basic/bin/runner run org.apache.distributedlog.basic.ConsoleProxyWriter 'inet!127.0.0.1:8000' basic-stream-2 +</pre> +</div> +<div class="section" id="check-the-results"> +<h4><a class="toc-backref" href="#id18">1.4.6 Check the results</a></h4> +<p>Example output from <cite>ConsoleProxyWriter</cite> and <cite>TailReader</cite>.</p> +<pre class="literal-block"> +// Output of `ConsoleProxyWriter` +May 08, 2016 10:27:41 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[inet] = com.twitter.finagle.InetResolver(com.twitter.finagle.InetResolver@756d7bba) +May 08, 2016 10:27:41 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[fixedinet] = com.twitter.finagle.FixedInetResolver(com.twitter.finagle.FixedInetResolver@1d2e91f5) +May 08, 2016 10:27:41 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[neg] = com.twitter.finagle.NegResolver$(com.twitter.finagle.NegResolver$@5c707aca) +May 08, 2016 10:27:41 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[nil] = com.twitter.finagle.NilResolver$(com.twitter.finagle.NilResolver$@5c8d932f) +May 08, 2016 10:27:41 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[fail] = com.twitter.finagle.FailResolver$(com.twitter.finagle.FailResolver$@52ba2221) +May 08, 2016 10:27:41 AM com.twitter.finagle.Init$$anonfun$1 apply$mcV$sp +[dlog] > test-proxy-writer +[dlog] > + + +// Output of `TailReader` +Opening log stream basic-stream-2 +Log stream basic-stream-2 is empty. +Wait for records starting from DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0} +Received record DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0} +""" +test-proxy-writer +""" +</pre> +</div> +<div class="section" id="attempt-a-second-writer"> +<h4><a class="toc-backref" href="#id19">1.4.7 Attempt a second writer</a></h4> +<p>Open another terminal to run <cite>ConsoleProxyWriter</cite>. The write should succeed as write proxy is able to accept +fan-in writes. Please checkout section <cite>Considerations</cite> to see the difference between <strong>Write Ordering</strong> and +<strong>Read Ordering</strong>.</p> +<pre class="literal-block"> +May 08, 2016 10:31:54 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[inet] = com.twitter.finagle.InetResolver(com.twitter.finagle.InetResolver@756d7bba) +May 08, 2016 10:31:54 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[fixedinet] = com.twitter.finagle.FixedInetResolver(com.twitter.finagle.FixedInetResolver@1d2e91f5) +May 08, 2016 10:31:54 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[neg] = com.twitter.finagle.NegResolver$(com.twitter.finagle.NegResolver$@5c707aca) +May 08, 2016 10:31:54 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[nil] = com.twitter.finagle.NilResolver$(com.twitter.finagle.NilResolver$@5c8d932f) +May 08, 2016 10:31:54 AM com.twitter.finagle.BaseResolver$$anonfun$resolvers$1 apply +INFO: Resolver[fail] = com.twitter.finagle.FailResolver$(com.twitter.finagle.FailResolver$@52ba2221) +May 08, 2016 10:31:54 AM com.twitter.finagle.Init$$anonfun$1 apply$mcV$sp +[dlog] > test-write-proxy-message-2 +[dlog] > +</pre> +</div> +</div> +</div> + + +</div> + + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved. + </p> + <p class="text-center"> + <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + <script> + (function () { + 'use strict'; + anchors.options.placement = 'right'; + anchors.add(); + })(); +</script> + + </body> + +</html>
