http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/1bd00e9a/architecture/main.html ---------------------------------------------------------------------- diff --git a/architecture/main.html b/architecture/main.html deleted file mode 100644 index 1321766..0000000 --- a/architecture/main.html +++ /dev/null @@ -1,669 +0,0 @@ -<!DOCTYPE html> - - -<html xmlns="http://www.w3.org/1999/xhtml";> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - - <title>Architecture — DistributedLog 1.0 documentation</title> - - <link rel="stylesheet" href="../_static/override.css" type="text/css" /> - <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> - <link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap-theme.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/featherlight.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/docbird.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/docbird-xs.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/jquery.rateyo.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/selection-sharer.css" type="text/css" /> - - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT: '../', - VERSION: '1.0', - COLLAPSE_INDEX: false, - FILE_SUFFIX: '.html', - HAS_SOURCE: true - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> - <script type="text/javascript" src="../_static/bootstrap-3.1.0/js/bootstrap.min.js"></script> - <script type="text/javascript" src="../_static/js/bootstrap-docbird.js"></script> - <script type="text/javascript" src="../_static/js/jquery-1.11.0.min.js"></script> - <script type="text/javascript" src="../_static/js/jquery-fix.js"></script> - <script type="text/javascript" src="../_static/js/featherlight.min.js"></script> - <script type="text/javascript" src="../_static/js/ifvisible.js"></script> - <script type="text/javascript" src="../_static/js/timeme.js"></script> - <script type="text/javascript" src="../_static/js/jquery.rateyo.min.js"></script> - <script type="text/javascript" src="../_static/js/js.cookie.js"></script> - <link rel="shortcut icon" href="../_static/docbird.ico"/> - <link rel="top" title="DistributedLog 1.0 documentation" href="../index.html" /> - <link rel="next" title="Detail Design" href="../design/main.html" /> - <link rel="prev" title="Considerations" href="../considerations/main.html" /> -<meta charset='utf-8'> -<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'> -<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'> -<meta name="apple-mobile-web-app-capable" content="yes"> - -<meta property="docbird:project" content="DistributedLog" /> - - </head> - <body> -<div class="navbar navbar-default navbar-fixed-top" role="navigation"> - <div class="container-fluid"> - <div class="row db-header"> - <div class="col-sm-3 col-md-3 col-lg-3 hidden-xs db-header-controls"> - <a href="/" alt="Back to Docbird"> - <div class="db-home-button"> - <span class="glyphicon glyphicon-home"></span> - </div> -</a> - -<form action="../search.html" method="get" class="db-searchbox-form"> - <div class="form-group"> - <input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" /> - </div> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> -</form> - - </div> - <div class="col-sm-7 col-md-7 col-lg-7 col-xs-12 db-header-info"> - <div class="visible-xs"> - <a href="/" alt="Back to Docbird"> - <div class="db-home-button"> - <span class="glyphicon glyphicon-home"></span> - </div> -</a> - </div> - <div class="visible-xs db-xs-menu-button"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#db-xs-menu"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> -</div> - </div> - <div class="db-header-projectname"> - <h1><a href="../index.html">DistributedLog</a></h1> - </div> - </div> - </div> - <div class="row db-xs-menu hidden-sm hidden-md hidden-lg - collapse" id="db-xs-menu"> - -<form action="../search.html" method="get" class="db-searchbox-form"> - <div class="form-group"> - <input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" /> - </div> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> -</form> - - <div class="db-toc" role="complementary"> - <ul class="current"> -<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../basics/main.html">Getting Started</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../basics/introduction.html">Introduction</a></li> -<li class="toctree-l2"><a class="reference internal" href="../basics/quickstart.html">Quick Start</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li> -</ul> -</li> -<li class="toctree-l1 current"><a class="current reference internal" href="">Architecture</a><ul> -<li class="toctree-l2"><a class="reference internal" href="#data-model">Data Model</a></li> -<li class="toctree-l2"><a class="reference internal" href="#software-stack">Software Stack</a></li> -<li class="toctree-l2"><a class="reference internal" href="#lifecyle-of-records">Lifecyle of records</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment & Administration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup & Deployment</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li> -<li class="toctree-l1"><a class="reference internal" href="../references/main.html">References</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../references/configuration.html">Configuration Settings</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/metrics.html">Metrics</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/features.html">Features</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> -</ul> - - </div> - </div> - </div> -</div> -<div class="container"> - <div class="row"> - <div style="z-index: 1" class="col-xs-12 col-sm-12 col-md-12 col-lg-12"> - <style> - .overflow-container { - display: none; - } - .overflow-toggle { - text-decoration: none; - border-bottom: none; - border-radius: 4px; - border: 1px solid #eee; - padding: 1px 3px 3px; - color: #888; - font-weight: normal; - background-color: linen; - line-height: 1.85em; - cursor: pointer; - } - .overflow-toggle:hover { - color: #333; - border-color: #ccc; - background-color: beige; - } -</style> -<script> -$(function(){ - $('.overflow-toggle').on('click', function(){ - $(this).next().toggle(); - }); -}); -</script> -<div class="db-project-header-container"> - <div class="row"> - - <div class="db-project-info col-lg-12 col-md-12 col-sm-12 col-xs-12"> - <h1> - <a href="../index.html">DistributedLog</a> - - </h1> - - <div class="db-code-link"> - <a href="[email protected]:twitter/distributedlog.git/tree/master/" target="_blank">[email protected]:twitter/distributedlog.git/tree/master/</a> - </div> - - - </div> - </div> - - <div class="row db-project-links-row"> - <div class=" col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <div class="db-hashtag-container"> - - <span class="db-project-link-label">OWNERS</span> - - <em>None</em> - - - </div> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <div class="db-hashtag-container"> - - <span class="db-project-link-label">TAGS</span> - - <em><a class="db-hashtag" href="/?q=tags:%23uses_maven">#uses_maven</a></em> - - - </div> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <span class="db-project-link-label">HEALTH</span> - - <h3 style="margin-top: 0"> -<!-- <a href="/techdocs/checklist.html" class="label label-success">--> - <a href="/report/distributedlog" class=""> - 9.0 / 10 - <span style="margin-left: .25em" class="glyphicon glyphicon-ok"></span> - </a> - - </h3> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <span class="db-project-link-label">RATING</span> - <div id="rateYo"></div> - </div> - </div> - -</div> - </div> - <div class="col-xs-12 col-sm-8 col-md-8 col-lg-8"> - <div class="db-content-body"> - - <div class="section" id="architecture"> -<h1>Architecture<a class="headerlink" href="#architecture" title="Permalink to this headline">¶</a></h1> -<div class="section" id="data-model"> -<h2>Data Model<a class="headerlink" href="#data-model" title="Permalink to this headline">¶</a></h2> -<p>DistributedLog exposes the <cite>log stream</cite> as the unit of operations. A <cite>log stream</cite> is a totally ordered, -immutable sequence of log records. A <cite>log record</cite> is a sequence of bytes. Log records are batched into <cite>entries</cite> -and written into <cite>log segments</cite>. Figure 1 illustrates the logical elements of a log stream.</p> -<div class="figure align-center"> -<img alt="../_images/datamodel.png" src="../_images/datamodel.png" /> -<p class="caption">Figure 1. Anatomy of a log stream</p> -</div> -<div class="section" id="log-segments"> -<h3>Log Segments<a class="headerlink" href="#log-segments" title="Permalink to this headline">¶</a></h3> -<p>Although the application views the log stream as a continuous sequence of log records, it is physically stored as -multiple <cite>log segments</cite>, where these segments are the unit of <cite>manageability</cite>. All the records in a log segment have -the same replication configuration. The log segments are allocated, distributed and stored in a <cite>log segment store</cite>. -As records are written to the log stream, DistributedLog switches to a new log segment based on a configured <cite>rolling policy</cite>. -The rolling policy can be <cite>time-based</cite> i.e. based on a configured period of time (e.g. every 2 hours) or <cite>size-based</cite> -i.e. based on a maximum log segment size (e.g. every 128MB). This allows the log segments to be distributed evenly -across all the storage nodes. This helps evenly spread read traffic to avoid hot spots in the cluster.</p> -<p>A log segment is also the unit of data retention. Log segments are deleted either by explicitly truncation or expiration. -Old data is garbage collected by the log segment store once the log segments are deleted.</p> -</div> -<div class="section" id="log-sequence-numbers"> -<h3>Log Sequence Numbers<a class="headerlink" href="#log-sequence-numbers" title="Permalink to this headline">¶</a></h3> -<p><cite>Log records</cite> are written sequentially into a log stream, and assigned a unique sequence number called <cite>DLSN</cite> -(DistributedLog Sequence Number). A DLSN is comprised of 3 components: a <cite>Log Segment Sequence Number</cite> (LSSN), -the sequence number of the log segment that the record belongs to, an <cite>Entry ID</cite> (EID) - the entry id in this log segment -that the record is in, and a <cite>Slot ID</cite> (SID) - the slot within the entry. Records can be ordered by DLSN.</p> -<p>Besides DLSN, an application can assign a <cite>Transaction ID</cite>, a non-decreasing positive 64-bit integer, to each log record it writes. -This facilitates application-specific sequencing of records and positioning of the reader. For example, a common use of the transaction ID -is to store the timestamp of when the log record was added to the log stream. This transaction ID can then be used to rewind to a specific -time in analytics applications.</p> -</div> -<div class="section" id="namespace"> -<h3>Namespace<a class="headerlink" href="#namespace" title="Permalink to this headline">¶</a></h3> -<p>Log streams that belong to same application are usually categorized and managed under a <cite>namespace</cite>. A <cite>namespace</cite> is used by applications -to locate where the log streams are. Applications can <cite>create</cite> and <cite>delete</cite> streams under a namespace, and <cite>truncate</cite> a stream to given ID.</p> -</div> -</div> -<div class="section" id="software-stack"> -<h2>Software Stack<a class="headerlink" href="#software-stack" title="Permalink to this headline">¶</a></h2> -<p>The software stack is shown in Figure 2. The software stack is divided into three layers, with each layer is responsible for -different features of DistributedLog. These layers are <cite>Persistent Storage</cite>, <cite>DistributedLog Core</cite> and <cite>Stateless Serving</cite>.</p> -<div class="figure align-center"> -<img alt="../_images/softwarestack.png" src="../_images/softwarestack.png" /> -<p class="caption">Figure 2. DistributedLog Software Stack</p> -</div> -<div class="section" id="persistent-storage"> -<h3>Persistent Storage<a class="headerlink" href="#persistent-storage" title="Permalink to this headline">¶</a></h3> -<p>DistributedLog provides the core features - <cite>durability</cite>, <cite>availability</cite> and <cite>consistency</cite> through the storage layer. -The main components in storage layer are <cite>Log Segment Store</cite>, <cite>Cold Storage</cite> and <cite>Metadata Store</cite>.</p> -<div class="section" id="log-segment-store"> -<h4>Log Segment Store<a class="headerlink" href="#log-segment-store" title="Permalink to this headline">¶</a></h4> -<p>The Log segment store is responsible for storing the log segments as they are created and ensure they are durably replicated. -We use <cite>Apache BookKeeper</cite> as the log segment store. BookKeeper helps us achieve low tail latencies for writes and reads as well as -low delivery latency which is the end to end latency from when the record is generated until it is read by the reader - because bookkeeper's -storage layout is optimized for I/O isolation for log workloads.</p> -<p>In addition to storage layout optimization, the log segment store (via BookKeeper) also provides a built-in <cite>fencing</cite> mechanism for -achieving strong consistency among multiple writers. We will discuss more about consistency in section <cite>Design Details</cite>.</p> -</div> -<div class="section" id="cold-storage"> -<h4>Cold Storage<a class="headerlink" href="#cold-storage" title="Permalink to this headline">¶</a></h4> -<p>The data in the log segment store is eventually moved to a <cite>cold storage</cite>. Cold storage allows cost efficient storage of large volumes -of log segments for extended period of time. Applications many want to have access to old data for application error recovery or debugging. -As log segments are completed, they are proactively copied over to the cold storage, thereby providing a backup for disaster recovery or an -operation error. We use HDFS as our cold storage.</p> -</div> -<div class="section" id="metadata-store"> -<h4>Metadata Store<a class="headerlink" href="#metadata-store" title="Permalink to this headline">¶</a></h4> -<p>The metadata in DistributedLog consists of the mapping from log streams to their constituent log segments as well as each log segmentâs metadata. -The log segment metadata includes the <cite>log segment ID</cite>, <cite>start and end transaction IDs</cite>, <cite>completion time</cite>, and its <cite>status</cite>. The metadata store -is required to provide metadata operations such as consistent read and write ordering to guarantee metadata consistency in the event of failures. -Also the metadata store should provide a notification mechanism to support streaming reads. We use ZooKeeper as the metadata store, because it is -a strongly consistent data store which provides versioned updates, strong ordering and notifications using watches.</p> -</div> -</div> -<div class="section" id="distributedlog-core"> -<h3>DistributedLog Core<a class="headerlink" href="#distributedlog-core" title="Permalink to this headline">¶</a></h3> -<p>DistributedLog builds its core functionality on top of the log segment store and the metadata store. It provides the core data model of log streams -and its naming system, and provides a <cite>single-writer-multiple-reader</cite> access model.</p> -<p>Writers write data into the logs of their choice. Writers sequence log records written to the log streams. Therefore there is only one active log -segment for a given log stream at a time. Correctness and consistency are guaranteed using a fencing mechanism in the log segment store and -versioned metadata updates to prevent two writers from writing to the same log stream after a network partition.</p> -<p>Reading from a log stream starts by <cite>positioning</cite> a reader on a log record by specifying either a DLSN or a Transaction Id. Once a reader has been -positioned, it receives all the log records in increasing order of the sequence numbers and each record is delivered exactly once. It is up to -individual applications to choose an appropriate mechanism to record readers positions and provide this position when a new reader session begins -(e.g restart from a failure). Applications can choose the appropriate method for storing positions based on the desired processing semantics.</p> -</div> -<div class="section" id="stateless-serving"> -<h3>Stateless Serving<a class="headerlink" href="#stateless-serving" title="Permalink to this headline">¶</a></h3> -<p>A stateless serving layer is built on top of the storage layer to support large number of writers and readers. The serving layer includes <cite>Write Proxy</cite> -and <cite>Read Proxy</cite>. <cite>Write Proxy</cite> manages the ownerships of the log streams, forwards the write requests to storage via the core library and handles -load balancing and failover. It allows sequencing writes from many clients (aka <cite>Fan-in</cite>). <cite>Read Proxy</cite> caches log records for multiple readers consuming -the same log stream.</p> -<div class="section" id="ownership-tracker"> -<h4>Ownership Tracker<a class="headerlink" href="#ownership-tracker" title="Permalink to this headline">¶</a></h4> -<p><cite>Ownership Tracker</cite> tracks the liveness of the owners of the log streams and fails over the ownership to other healthy write proxies when the current -owner becomes unavailable. Since we already use zookeeper for metadata storage, we also use zookeeper for tracking the liveness of write proxies using -<cite>ephemeral znodes</cite> and failover the ownership when zookeeper session expires.</p> -</div> -<div class="section" id="routing-service"> -<h4>Routing Service<a class="headerlink" href="#routing-service" title="Permalink to this headline">¶</a></h4> -<p>Since readers read committed data and are strict followers, the read proxies do not have to track ownership of log streams. We use consistent hashing -as a routing mechanism to route the readers to corresponding read proxies.</p> -<p>Applications can either use a thin client that talks to the serving tier to access DistributedLog or embed the core library to talk to the storage directly -when they require strict write ordering. Applications choose partitioning strategies and track their reader positions based on their specific requirements.</p> -</div> -</div> -</div> -<div class="section" id="lifecyle-of-records"> -<h2>Lifecyle of records<a class="headerlink" href="#lifecyle-of-records" title="Permalink to this headline">¶</a></h2> -<p>Figure 3 illustrates the lifecycle of a log record in DistributedLog as it flows from writers to readers and we discuss how different layers interact with -each other.</p> -<div class="figure align-center"> -<img alt="../_images/requestflow.png" src="../_images/requestflow.png" /> -<p class="caption">Figure 3. Lifecycle of a record</p> -</div> -<p>The application constructs the log records and initiates write requests (step 1). The write requests will be forwarded to the write proxy that is the master -of the log stream. The master writer proxy will write the records in the log streamâs transmit buffer. Based on the configured transmit policy, records in -the transmit buffer will be transmitted as a batched entry to log segment store (step 2). Application can trade latency for throughput by transmitting -<cite>immediately</cite> (lowest latency), <cite>periodically</cite> (grouping records that appear within the transmit period) or when transmit buffer has accumulated more than -<cite>max-outstanding bytes</cite>.</p> -<p>The batched entry is transmitted to multiple bookies (storage nodes in bookkeeper cluster) in parallel (step 3). The log segment store will respond back to -writer once the entry is persisted durably on disk. Once the write proxy receives confirmation of durability from a quorum of bookies, it will send an -acknowledgement to the application (step 4).</p> -<p>Although the writer knows that the record is guaranteed to be persisted and replicated in bookkeeper. Individual bookies do not necessarily know that the -consensus agreement has been reached for this record. The writer must therefore record a <cite>commit</cite> to make this record visible to all the readers. -This <cite>commit</cite> can piggyback on the next batch of records from the application. If no new application records are received within the specified SLA for -persistence, the writer will issue a special <cite>control log record</cite> notifying the log segment store that the record can now be made visible to readers (step 5).</p> -<p>The readersâ request that is waiting for new data using <cite>long polling</cite> will now receive the recent committed log records (step 6). Speculative long poll reads will be sent to other replicas to archieve predictable low 99.9% percentile latency (step 7).</p> -<p>The log records will be cached in read proxies (step 8) for fanout readers. The read clients also use similar long poll read mechanism to read data from read proxies (step 9).</p> -</div> -</div> - - - </div> - </div> - <div class="hidden-xs col-sm-3 col-md-3 col-md-offset-1 col-lg-3 db-sidebar"> - - <div class="db-toc" role="complementary"> - <ul class="current"> - <li class="toctree-l0 current"><a class="current reference internal" href="../index.html">DistributedLog</a></li> - </ul> - <ul class="current"> -<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../basics/main.html">Getting Started</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../basics/introduction.html">Introduction</a></li> -<li class="toctree-l2"><a class="reference internal" href="../basics/quickstart.html">Quick Start</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li> -</ul> -</li> -<li class="toctree-l1 current"><a class="current reference internal" href="">Architecture</a><ul> -<li class="toctree-l2"><a class="reference internal" href="#data-model">Data Model</a></li> -<li class="toctree-l2"><a class="reference internal" href="#software-stack">Software Stack</a></li> -<li class="toctree-l2"><a class="reference internal" href="#lifecyle-of-records">Lifecyle of records</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment & Administration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup & Deployment</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li> -<li class="toctree-l1"><a class="reference internal" href="../references/main.html">References</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../references/configuration.html">Configuration Settings</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/metrics.html">Metrics</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/features.html">Features</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> -</ul> - - <span id="last"></span> - </div> - - </div> - <!-- <div id="slidebox"> --> - <!-- <button id="slidebox_close" type="button" class="close">×</button> --> - <!-- <p>Rate This Page</p> --> - <!-- <div id="rateYo"></div> --> - <!-- <p>Comment</p> - <input type="text" name="comment"></input> - <button>Submit</button> --> - <!-- </div> --> - </div> -</div> -<footer class="footer"> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-10 col-md-offset-1"> - <p class="pull-right"> - <a href="#">Back to top</a> - - <br/> - -<div id="sourcelink"> - <a href="[email protected]:twitter/distributedlog.git/tree/master/docs/architecture/main.rst" - rel="nofollow">Source</a> - - <a href="../_sources/architecture/main.txt" - rel="nofollow">Raw</a> - <a href="../__docbird-build.log" - rel="nofollow">Build Log</a> - <a href="/report/stats/distributedlog:distributedlog" - rel="nofollow">Stats</a> -</div> - </p> - <p> - Built and hosted by <a href="/">DocBird</a>. - </p> - </div> - </div> - </div> -</footer> -<script type="text/javascript" src="../_static/js/docbird.js"></script> -<script type="text/javascript"> - var _gaq = _gaq || []; - _gaq.push(['_setAccount', 'UA-30775-8']); - _gaq.push(['_trackPageview']); - - (function() { - var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; - ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; - var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); - })(); -</script> -<!-- <script type="text/javascript" src="//s/d41d8cd98f00b204e9800998ecf8427e/en_US-tbnx1s-1988229788/6163/97/1.4.3/_/download/batch/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector.js?collectorId=e62237fc"></script> ---> - -<script type="text/javascript"> - $(document).ready(function () { - // track user activity time (from https://github.com/jasonzissman/TimeMe.js) - TimeMe.setIdleDurationInSeconds(30); - TimeMe.setCurrentPageName("my-home-page"); - TimeMe.initialize(); - - // record page visit event when user leaves the page - window.onbeforeunload = function (event) { - xmlhttp=new XMLHttpRequest(); - xmlhttp.withCredentials = true; - xmlhttp.open("POST", "/event/distributedlog:distributedlog/visit", false); - xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); - var event_data = { - total_time_reading: TimeMe.getTimeOnCurrentPageInSeconds(), - page: window.location.href - }; - //alert("send: " + $.param(event_data)); - xmlhttp.send($.param(event_data)); - }; - - // ask user for page rating after 20 seconds - // setTimeout(function(){ - // alert("Rate this page!"); - // }, 20000); - }); -</script> -<!-- <style> -#slidebox{ - width: 250px; - height: 90px; - padding: 10px; - background-color: #fff; - border: 1px solid #ccc; - position: fixed; - bottom: 3px; - right: -280px; - z-index: 1; -} -#slidebox .close{ - margin-top: -5px; - opacity: 0.5; -} -#slidebox .close:hover{ - opacity: 0.7; -} -</style> --> -<script type="text/javascript"> -$(function() { - // $(window).scroll(function(){ - // var distanceTop = $('#last').offset().top - $(window).height(); - - // if ($(window).scrollTop() > distanceTop) - // $('#slidebox').animate({'right':'3px'},300); - // else - // $('#slidebox').stop(true).animate({'right':'-280px'},100); - // }); - - // $('#slidebox .close').bind('click',function(){ - // $(this).parent().remove(); - // }); - - $("#rateYo").rateYo({ - normalFill: "#A0A0A0", - halfStar: true, - rating: (Cookies.get('docbird.rating.distributedlog.distributedlog') || 0.0) - }).on("rateyo.set", function (e, data) { - var event_data = { - comment: '', // see todo note below - rating: data.rating, - page: window.location.href - }; - Cookies.get('docbird.rating.distributedlog.distributedlog', data.rating) - $.post('/event/distributedlog:distributedlog/rating', event_data) - // xmlhttp=new XMLHttpRequest(); - // xmlhttp.withCredentials = true; - // var event_data = { - // comment: '', // see todo note below - // rating: data.rating, - // page: window.location.href - // }; - // xmlhttp.open("GET", "/event/distributedlog/rating?" + $.param(event_data), false); - // xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); - // // todo: implement comment form in rating slide out, - // // and instead of hooking this event, include a submit button, - // // and read the rating with rating() method - - // // alert("send: " + $.param(event_data)); - // xmlhttp.send(); - - }); - -}); -</script> -<script src="_static/js/selection-sharer.js"></script> -<script> -$('.db-content-body').selectionSharer(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/1bd00e9a/basics/introduction.html ---------------------------------------------------------------------- diff --git a/basics/introduction.html b/basics/introduction.html deleted file mode 100644 index d10db93..0000000 --- a/basics/introduction.html +++ /dev/null @@ -1,639 +0,0 @@ -<!DOCTYPE html> - - -<html xmlns="http://www.w3.org/1999/xhtml";> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - - <title>Introduction — DistributedLog 1.0 documentation</title> - - <link rel="stylesheet" href="../_static/override.css" type="text/css" /> - <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> - <link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap-theme.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/featherlight.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/docbird.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/docbird-xs.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/jquery.rateyo.min.css" type="text/css" /> - <link rel="stylesheet" href="../_static/css/selection-sharer.css" type="text/css" /> - - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT: '../', - VERSION: '1.0', - COLLAPSE_INDEX: false, - FILE_SUFFIX: '.html', - HAS_SOURCE: true - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> - <script type="text/javascript" src="../_static/bootstrap-3.1.0/js/bootstrap.min.js"></script> - <script type="text/javascript" src="../_static/js/bootstrap-docbird.js"></script> - <script type="text/javascript" src="../_static/js/jquery-1.11.0.min.js"></script> - <script type="text/javascript" src="../_static/js/jquery-fix.js"></script> - <script type="text/javascript" src="../_static/js/featherlight.min.js"></script> - <script type="text/javascript" src="../_static/js/ifvisible.js"></script> - <script type="text/javascript" src="../_static/js/timeme.js"></script> - <script type="text/javascript" src="../_static/js/jquery.rateyo.min.js"></script> - <script type="text/javascript" src="../_static/js/js.cookie.js"></script> - <link rel="shortcut icon" href="../_static/docbird.ico"/> - <link rel="top" title="DistributedLog 1.0 documentation" href="../index.html" /> - <link rel="up" title="Getting Started" href="main.html" /> - <link rel="next" title="Quick Start" href="quickstart.html" /> - <link rel="prev" title="Getting Started" href="main.html" /> -<meta charset='utf-8'> -<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'> -<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'> -<meta name="apple-mobile-web-app-capable" content="yes"> - -<meta property="docbird:project" content="DistributedLog" /> - - </head> - <body> -<div class="navbar navbar-default navbar-fixed-top" role="navigation"> - <div class="container-fluid"> - <div class="row db-header"> - <div class="col-sm-3 col-md-3 col-lg-3 hidden-xs db-header-controls"> - <a href="/" alt="Back to Docbird"> - <div class="db-home-button"> - <span class="glyphicon glyphicon-home"></span> - </div> -</a> - -<form action="../search.html" method="get" class="db-searchbox-form"> - <div class="form-group"> - <input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" /> - </div> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> -</form> - - </div> - <div class="col-sm-7 col-md-7 col-lg-7 col-xs-12 db-header-info"> - <div class="visible-xs"> - <a href="/" alt="Back to Docbird"> - <div class="db-home-button"> - <span class="glyphicon glyphicon-home"></span> - </div> -</a> - </div> - <div class="visible-xs db-xs-menu-button"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#db-xs-menu"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> -</div> - </div> - <div class="db-header-projectname"> - <h1><a href="../index.html">DistributedLog</a></h1> - </div> - </div> - </div> - <div class="row db-xs-menu hidden-sm hidden-md hidden-lg - collapse" id="db-xs-menu"> - -<form action="../search.html" method="get" class="db-searchbox-form"> - <div class="form-group"> - <input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" /> - </div> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> -</form> - - <div class="db-toc" role="complementary"> - <ul class="current"> -<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li> -</ul> -</li> -<li class="toctree-l1 current"><a class="reference internal" href="main.html">Getting Started</a><ul class="current"> -<li class="toctree-l2 current"><a class="current reference internal" href="">Introduction</a></li> -<li class="toctree-l2"><a class="reference internal" href="quickstart.html">Quick Start</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../architecture/main.html">Architecture</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#data-model">Data Model</a></li> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#software-stack">Software Stack</a></li> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#lifecyle-of-records">Lifecyle of records</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment & Administration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup & Deployment</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li> -<li class="toctree-l1"><a class="reference internal" href="../references/main.html">References</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../references/configuration.html">Configuration Settings</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/metrics.html">Metrics</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/features.html">Features</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> -</ul> - - </div> - </div> - </div> -</div> -<div class="container"> - <div class="row"> - <div style="z-index: 1" class="col-xs-12 col-sm-12 col-md-12 col-lg-12"> - <style> - .overflow-container { - display: none; - } - .overflow-toggle { - text-decoration: none; - border-bottom: none; - border-radius: 4px; - border: 1px solid #eee; - padding: 1px 3px 3px; - color: #888; - font-weight: normal; - background-color: linen; - line-height: 1.85em; - cursor: pointer; - } - .overflow-toggle:hover { - color: #333; - border-color: #ccc; - background-color: beige; - } -</style> -<script> -$(function(){ - $('.overflow-toggle').on('click', function(){ - $(this).next().toggle(); - }); -}); -</script> -<div class="db-project-header-container"> - <div class="row"> - - <div class="db-project-info col-lg-12 col-md-12 col-sm-12 col-xs-12"> - <h1> - <a href="../index.html">DistributedLog</a> - - </h1> - - <div class="db-code-link"> - <a href="[email protected]:twitter/distributedlog.git/tree/master/" target="_blank">[email protected]:twitter/distributedlog.git/tree/master/</a> - </div> - - - </div> - </div> - - <div class="row db-project-links-row"> - <div class=" col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <div class="db-hashtag-container"> - - <span class="db-project-link-label">OWNERS</span> - - <em>None</em> - - - </div> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <div class="db-hashtag-container"> - - <span class="db-project-link-label">TAGS</span> - - <em><a class="db-hashtag" href="/?q=tags:%23uses_maven">#uses_maven</a></em> - - - </div> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <span class="db-project-link-label">HEALTH</span> - - <h3 style="margin-top: 0"> -<!-- <a href="/techdocs/checklist.html" class="label label-success">--> - <a href="/report/distributedlog" class=""> - 9.0 / 10 - <span style="margin-left: .25em" class="glyphicon glyphicon-ok"></span> - </a> - - </h3> - </div> - <div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column"> - <span class="db-project-link-label">RATING</span> - <div id="rateYo"></div> - </div> - </div> - -</div> - </div> - <div class="col-xs-12 col-sm-8 col-md-8 col-lg-8"> - <div class="db-content-body"> - - <div class="section" id="introduction"> -<h1>Introduction<a class="headerlink" href="#introduction" title="Permalink to this headline">¶</a></h1> -<p>DistributedLog (DL) is a high performance replicated log service. -It offers durability, replication and strong consistency, which provides a fundamental building block -for building reliable distributed systems, e.g replicated-state-machines, general pub/sub systems, -distributed databases, distributed queues and etc.</p> -<p>DistributedLog maintains sequences of records in categories called <em>Logs</em> (aka <em>Log Streams</em>). -The processes that write records to a DL log are <em>writers</em>, while the processes that read -from logs and process the records are <em>readers</em>.</p> -<div class="figure align-center"> -<img alt="../_images/softwarestack1.png" src="../_images/softwarestack1.png" /> -<p class="caption">Figure 1. DistributedLog Software Stack</p> -</div> -<div class="section" id="logs"> -<h2>Logs<a class="headerlink" href="#logs" title="Permalink to this headline">¶</a></h2> -<p>A <strong>log</strong> is an ordered, immutable sequence of <em>log records</em>.</p> -<div class="figure align-center"> -<img alt="../_images/datamodel1.png" src="../_images/datamodel1.png" /> -<p class="caption">Figure 2. Anatomy of a log stream</p> -</div> -<div class="section" id="log-records"> -<h3>Log Records<a class="headerlink" href="#log-records" title="Permalink to this headline">¶</a></h3> -<p>Each <strong>log record</strong> is a sequence of bytes. -<strong>Log records</strong> are written sequentially into a <em>log stream</em>, and will be assigned with -a unique sequence number <em>called</em> <strong>DLSN</strong> (DistributedLog Sequence Number). Besides <em>DLSN</em>, -applications could assign its own sequence number while constructing log records. The -application defined sequence number is called <strong>TransactionID</strong> (<em>txid</em>). Either <em>DLSN</em> -or <em>TransactionID</em> could be used for positioning readers to start reading from a specific -<em>log record</em>.</p> -</div> -<div class="section" id="log-segments"> -<h3>Log Segments<a class="headerlink" href="#log-segments" title="Permalink to this headline">¶</a></h3> -<p>A <strong>log</strong> is broken down into <em>segments</em>, which each log segment contains its subset of -records. <strong>Log segments</strong> are distributed and stored in a log segment store (e.g Apache BookKeeper). -DistributedLog rolls the log segments based on configured rolling policy - either a configurable -period of time (e.g. every 2 hours) or a configurable maximum size (e.g. every 128MB). -So the data of logs will be divided into equal-sized <em>log segments</em> and distributed evenly -across log segment storage nodes. It allows the log to scale beyond a size that will fit on -a single server and also spread read traffic among the cluster.</p> -<p>The data of logs will either be kept forever until application <em>explicitly</em> truncates or be retained -for a configurable period of time. <strong>Explicit Truncation</strong> is useful for building replicated -state machines such as distributed databases. They usually require strong controls over when -the data could be truncated. <strong>Time-based Retention</strong> is useful for real-time analytics. They only -care about the data within a period of time.</p> -</div> -<div class="section" id="namespaces"> -<h3>Namespaces<a class="headerlink" href="#namespaces" title="Permalink to this headline">¶</a></h3> -<p>The <em>log streams</em> belong to same organization are usually categorized and managed under -a <strong>namespace</strong>. A DL <strong>namespace</strong> is basically for applications to locate where the -<em>log streams</em> are. Applications could <em>create</em> and <em>delete</em> streams under a namespace, -and also be able to <em>truncate</em> a stream to given sequence number (either <em>DLSN</em> or <em>TransactionID</em>).</p> -</div> -</div> -<div class="section" id="writers"> -<h2>Writers<a class="headerlink" href="#writers" title="Permalink to this headline">¶</a></h2> -<p>Writers write data into the logs of their choice. All the records are -appended into the logs in order. The sequencing is done by the writer, -which means there is only one active writer for a log at a given time. -DL guarantees correctness when two writers attempt writing to -to a same log when network partition happens - via fencing mechanism -in log segment store.</p> -<p>The log writers are served and managed in a service tier called <em>Write Proxy</em>. -The <em>Write Proxy</em> is used for accepting fan-in writes from large number -of clients. Details on <strong>Fan-in and Fan-out</strong> can be found further into this doc.</p> -</div> -<div class="section" id="readers"> -<h2>Readers<a class="headerlink" href="#readers" title="Permalink to this headline">¶</a></h2> -<p>Readers read records from the logs of their choice, starting from a provided -position. The provided position could be either <em>DLSN</em> or <em>TransactionID</em>. -The readers will read records in strict order from the logs. Different readers -could read records starting from different positions in a same log.</p> -<p>Unlike other pub/sub systems, DistributedLog doesn't record/manage readers' positions. -It leaves the tracking responsibility to applications, as different applications -might have different requirements on tracking and coordinating positions. It is hard -to get it right with a single approach. For example, distributed databases might store -the reader positions along with SSTables, so they would resume applying transactions -from the positions stored in SSTables. Tracking reader positions could easily be done -in application level using various stores (e.g. ZooKeeper, FileSystem, or Key/Value Stores).</p> -<p>The log records could be cached in a service tier called <em>Read Proxy</em>, to serve -large number of readers. Details on <strong>Fan-in and Fan-out</strong> can be found further into this doc.</p> -</div> -<div class="section" id="fan-in-and-fan-out"> -<h2>Fan-in and Fan-out<a class="headerlink" href="#fan-in-and-fan-out" title="Permalink to this headline">¶</a></h2> -<p>The core of DistributedLog supports single-writer, multiple-readers semantics. The service layer -built on top of the <em>DistributedLog Core</em> to support large scale of number of writers and readers. -The service layer includes <strong>Write Proxy</strong> and <strong>Read Proxy</strong>. <strong>Write Proxy</strong> manages -the writers of logs and fail over them when machines are failed. It allows supporting -which don't care about the log ownership by aggregating writes from many sources (aka <em>Fan-in</em>). -<strong>Read Proxy</strong> optimize reader path by caching log records in cases where hundreds or -thousands of readers are consuming a same log stream.</p> -</div> -<div class="section" id="guarantees"> -<h2>Guarantees<a class="headerlink" href="#guarantees" title="Permalink to this headline">¶</a></h2> -<p>At a high-level, DistributedLog gives the following guarantees:</p> -<ul class="simple"> -<li>Records written by a writer to a log will be appended in the order they are written. That is, if a record <em>R1</em> is written by same writer as a record <em>R2</em>, <em>R1</em> will have a smaller sequence number than <em>R2</em>.</li> -<li>Readers will see records in same order they were written to the log.</li> -<li>All records were persisted on disks before acknowledges, to gurantee durability.</li> -<li>For a log with replication factor of N, DistributedLog tolerates up to N-1 server failures without losing any records appended to the log.</li> -</ul> -<p>More details on these guarantees are given in the design section of this documentation.</p> -</div> -</div> - - - </div> - </div> - <div class="hidden-xs col-sm-3 col-md-3 col-md-offset-1 col-lg-3 db-sidebar"> - - <div class="db-toc" role="complementary"> - <ul class="current"> - <li class="toctree-l0 current"><a class="current reference internal" href="../index.html">DistributedLog</a></li> - </ul> - <ul class="current"> -<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li> -<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li> -</ul> -</li> -<li class="toctree-l1 current"><a class="reference internal" href="main.html">Getting Started</a><ul class="current"> -<li class="toctree-l2 current"><a class="current reference internal" href="">Introduction</a></li> -<li class="toctree-l2"><a class="reference internal" href="quickstart.html">Quick Start</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li> -<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li> -<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../architecture/main.html">Architecture</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#data-model">Data Model</a></li> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#software-stack">Software Stack</a></li> -<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#lifecyle-of-records">Lifecyle of records</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li> -<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li> -<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment & Administration</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup & Deployment</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li> -<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li> -<li class="toctree-l1"><a class="reference internal" href="../references/main.html">References</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../references/configuration.html">Configuration Settings</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/metrics.html">Metrics</a></li> -<li class="toctree-l2"><a class="reference internal" href="../references/features.html">Features</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li> -<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul> -<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li> -</ul> -</li> -<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> -</ul> - - <span id="last"></span> - </div> - - </div> - <!-- <div id="slidebox"> --> - <!-- <button id="slidebox_close" type="button" class="close">×</button> --> - <!-- <p>Rate This Page</p> --> - <!-- <div id="rateYo"></div> --> - <!-- <p>Comment</p> - <input type="text" name="comment"></input> - <button>Submit</button> --> - <!-- </div> --> - </div> -</div> -<footer class="footer"> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-10 col-md-offset-1"> - <p class="pull-right"> - <a href="#">Back to top</a> - - <br/> - -<div id="sourcelink"> - <a href="[email protected]:twitter/distributedlog.git/tree/master/docs/basics/introduction.rst" - rel="nofollow">Source</a> - - <a href="../_sources/basics/introduction.txt" - rel="nofollow">Raw</a> - <a href="../__docbird-build.log" - rel="nofollow">Build Log</a> - <a href="/report/stats/distributedlog:distributedlog" - rel="nofollow">Stats</a> -</div> - </p> - <p> - Built and hosted by <a href="/">DocBird</a>. - </p> - </div> - </div> - </div> -</footer> -<script type="text/javascript" src="../_static/js/docbird.js"></script> -<script type="text/javascript"> - var _gaq = _gaq || []; - _gaq.push(['_setAccount', 'UA-30775-8']); - _gaq.push(['_trackPageview']); - - (function() { - var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; - ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; - var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); - })(); -</script> -<!-- <script type="text/javascript" src="//s/d41d8cd98f00b204e9800998ecf8427e/en_US-tbnx1s-1988229788/6163/97/1.4.3/_/download/batch/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector.js?collectorId=e62237fc"></script> ---> - -<script type="text/javascript"> - $(document).ready(function () { - // track user activity time (from https://github.com/jasonzissman/TimeMe.js) - TimeMe.setIdleDurationInSeconds(30); - TimeMe.setCurrentPageName("my-home-page"); - TimeMe.initialize(); - - // record page visit event when user leaves the page - window.onbeforeunload = function (event) { - xmlhttp=new XMLHttpRequest(); - xmlhttp.withCredentials = true; - xmlhttp.open("POST", "/event/distributedlog:distributedlog/visit", false); - xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); - var event_data = { - total_time_reading: TimeMe.getTimeOnCurrentPageInSeconds(), - page: window.location.href - }; - //alert("send: " + $.param(event_data)); - xmlhttp.send($.param(event_data)); - }; - - // ask user for page rating after 20 seconds - // setTimeout(function(){ - // alert("Rate this page!"); - // }, 20000); - }); -</script> -<!-- <style> -#slidebox{ - width: 250px; - height: 90px; - padding: 10px; - background-color: #fff; - border: 1px solid #ccc; - position: fixed; - bottom: 3px; - right: -280px; - z-index: 1; -} -#slidebox .close{ - margin-top: -5px; - opacity: 0.5; -} -#slidebox .close:hover{ - opacity: 0.7; -} -</style> --> -<script type="text/javascript"> -$(function() { - // $(window).scroll(function(){ - // var distanceTop = $('#last').offset().top - $(window).height(); - - // if ($(window).scrollTop() > distanceTop) - // $('#slidebox').animate({'right':'3px'},300); - // else - // $('#slidebox').stop(true).animate({'right':'-280px'},100); - // }); - - // $('#slidebox .close').bind('click',function(){ - // $(this).parent().remove(); - // }); - - $("#rateYo").rateYo({ - normalFill: "#A0A0A0", - halfStar: true, - rating: (Cookies.get('docbird.rating.distributedlog.distributedlog') || 0.0) - }).on("rateyo.set", function (e, data) { - var event_data = { - comment: '', // see todo note below - rating: data.rating, - page: window.location.href - }; - Cookies.get('docbird.rating.distributedlog.distributedlog', data.rating) - $.post('/event/distributedlog:distributedlog/rating', event_data) - // xmlhttp=new XMLHttpRequest(); - // xmlhttp.withCredentials = true; - // var event_data = { - // comment: '', // see todo note below - // rating: data.rating, - // page: window.location.href - // }; - // xmlhttp.open("GET", "/event/distributedlog/rating?" + $.param(event_data), false); - // xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); - // // todo: implement comment form in rating slide out, - // // and instead of hooking this event, include a submit button, - // // and read the rating with rating() method - - // // alert("send: " + $.param(event_data)); - // xmlhttp.send(); - - }); - -}); -</script> -<script src="_static/js/selection-sharer.js"></script> -<script> -$('.db-content-body').selectionSharer(); -</script> - </body> -</html>
