http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/12/04/Introducing-windows.html ---------------------------------------------------------------------- diff --git a/content/news/2015/12/04/Introducing-windows.html b/content/news/2015/12/04/Introducing-windows.html new file mode 100644 index 0000000..aa0834f --- /dev/null +++ b/content/news/2015/12/04/Introducing-windows.html @@ -0,0 +1,349 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Introducing Stream Windows in Apache Flink</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Introducing Stream Windows in Apache Flink</h1> + + <article> + <p>04 Dec 2015 by Fabian Hueske (<a href="https://twitter.com/fhueske";>@fhueske</a>)</p> + +<p>The data analysis space is witnessing an evolution from batch to stream processing for many use cases. Although batch can be handled as a special case of stream processing, analyzing never-ending streaming data often requires a shift in the mindset and comes with its own terminology (for example, âwindowingâ and âat-least-onceâ/âexactly-onceâ processing). This shift and the new terminology can be quite confusing for people being new to the space of stream processing. Apache Flink is a production-ready stream processor with an easy-to-use yet very expressive API to define advanced stream analysis programs. Flinkâs API features very flexible window definitions on data streams which let it stand out among other open source stream processors.</p> + +<p>In this blog post, we discuss the concept of windows for stream processing, present Flinkâs built-in windows, and explain its support for custom windowing semantics.</p> + +<h2 id="what-are-windows-and-what-are-they-good-for">What are windows and what are they good for?</h2> + +<p>Consider the example of a traffic sensor that counts every 15 seconds the number of vehicles passing a certain location. The resulting stream could look like:</p> + +<center> +<img src="/img/blog/window-intro/window-stream.png" style="width:75%;margin:15px" /> +</center> + +<p>If you would like to know, how many vehicles passed that location, you would simply sum the individual counts. However, the nature of a sensor stream is that it continuously produces data. Such a stream never ends and it is not possible to compute a final sum that can be returned. Instead, it is possible to compute rolling sums, i.e., return for each input event an updated sum record. This would yield a new stream of partial sums.</p> + +<center> +<img src="/img/blog/window-intro/window-rolling-sum.png" style="width:75%;margin:15px" /> +</center> + +<p>However, a stream of partial sums might not be what we are looking for, because it constantly updates the count and even more important, some information such as variation over time is lost. Hence, we might want to rephrase our question and ask for the number of cars that pass the location every minute. This requires us to group the elements of the stream into finite sets, each set corresponding to sixty seconds. This operation is called a <em>tumbling windows</em> operation.</p> + +<center> +<img src="/img/blog/window-intro/window-tumbling-window.png" style="width:75%;margin:15px" /> +</center> + +<p>Tumbling windows discretize a stream into non-overlapping windows. For certain applications it is important that windows are not disjunct because an application might require smoothed aggregates. For example, we can compute every thirty seconds the number of cars passed in the last minute. Such windows are called <em>sliding windows</em>.</p> + +<center> +<img src="/img/blog/window-intro/window-sliding-window.png" style="width:75%;margin:15px" /> +</center> + +<p>Defining windows on a data stream as discussed before is a non-parallel operation. This is because each element of a stream must be processed by the same window operator that decides which windows the element should be added to. Windows on a full stream are called <em>AllWindows</em> in Flink. For many applications, a data stream needs to be grouped into multiple logical streams on each of which a window operator can be applied. Think for example about a stream of vehicle counts from multiple traffic sensors (instead of only one sensor as in our previous example), where each sensor monitors a different location. By grouping the stream by sensor id, we can compute windowed traffic statistics for each location in parallel. In Flink, we call such partitioned windows simply <em>Windows</em>, as they are the common case for distributed streams. The following figure shows tumbling windows that collect two elements over a stream of <code>(sensorId, count)</code> pair elements.</p> + +<center> +<img src="/img/blog/window-intro/windows-keyed.png" style="width:75%;margin:15px" /> +</center> + +<p>Generally speaking, a window defines a finite set of elements on an unbounded stream. This set can be based on time (as in our previous examples), element counts, a combination of counts and time, or some custom logic to assign elements to windows. Flinkâs DataStream API provides concise operators for the most common window operations as well as a generic windowing mechanism that allows users to define very custom windowing logic. In the following we present Flinkâs time and count windows before discussing its windowing mechanism in detail.</p> + +<h2 id="time-windows">Time Windows</h2> + +<p>As their name suggests, time windows group stream elements by time. For example, a tumbling time window of one minute collects elements for one minute and applies a function on all elements in the window after one minute passed.</p> + +<p>Defining tumbling and sliding time windows in Apache Flink is very easy:</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// Stream of (sensorId, carCnt)</span> +<span class="k">val</span> <span class="n">vehicleCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="o">...</span> + +<span class="k">val</span> <span class="n">tumblingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> + <span class="c1">// key stream by sensorId</span> + <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="c1">// tumbling time window of 1 minute length</span> + <span class="o">.</span><span class="n">timeWindow</span><span class="o">(</span><span class="nc">Time</span><span class="o">.</span><span class="n">minutes</span><span class="o">(</span><span class="mi">1</span><span class="o">))</span> + <span class="c1">// compute sum over carCnt</span> + <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> + +<span class="k">val</span> <span class="n">slidingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> + <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="c1">// sliding time window of 1 minute length and 30 secs trigger interval</span> + <span class="o">.</span><span class="n">timeWindow</span><span class="o">(</span><span class="nc">Time</span><span class="o">.</span><span class="n">minutes</span><span class="o">(</span><span class="mi">1</span><span class="o">),</span> <span class="nc">Time</span><span class="o">.</span><span class="n">seconds</span><span class="o">(</span><span class="mi">30</span><span class="o">))</span> + <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span></code></pre></div> + +<p>There is one aspect that we havenât discussed yet, namely the exact meaning of â<em>collects elements for one minute</em>â which boils down to the question, â<em>How does the stream processor interpret time?</em>â.</p> + +<p>Apache Flink features three different notions of time, namely <em>processing time</em>, <em>event time</em>, and <em>ingestion time</em>.</p> + +<ol> + <li>In <strong>processing time</strong>, windows are defined with respect to the wall clock of the machine that builds and processes a window, i.e., a one minute processing time window collects elements for exactly one minute.</li> + <li>In <strong>event time</strong>, windows are defined with respect to timestamps that are attached to each event record. This is common for many types of events, such as log entries, sensor data, etc, where the timestamp usually represents the time at which the event occurred. Event time has several benefits over processing time. First of all, it decouples the program semantics from the actual serving speed of the source and the processing performance of system. Hence you can process historic data, which is served at maximum speed, and continuously produced data with the same program. It also prevents semantically incorrect results in case of backpressure or delays due to failure recovery. Second, event time windows compute correct results, even if events arrive out-of-order of their timestamp which is common if a data stream gathers events from distributed sources.</li> + <li><strong>Ingestion time</strong> is a hybrid of processing and event time. It assigns wall clock timestamps to records as soon as they arrive in the system (at the source) and continues processing with event time semantics based on the attached timestamps.</li> +</ol> + +<h2 id="count-windows">Count Windows</h2> + +<p>Apache Flink also features count windows. A tumbling count window of 100 will collect 100 events in a window and evaluate the window when the 100th element has been added.</p> + +<p>In Flinkâs DataStream API, tumbling and sliding count windows are defined as follows:</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// Stream of (sensorId, carCnt)</span> +<span class="k">val</span> <span class="n">vehicleCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="o">...</span> + +<span class="k">val</span> <span class="n">tumblingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> + <span class="c1">// key stream by sensorId</span> + <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="c1">// tumbling count window of 100 elements size</span> + <span class="o">.</span><span class="n">countWindow</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span> + <span class="c1">// compute the carCnt sum </span> + <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> + +<span class="k">val</span> <span class="n">slidingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> + <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="c1">// sliding count window of 100 elements size and 10 elements trigger interval</span> + <span class="o">.</span><span class="n">countWindow</span><span class="o">(</span><span class="mi">100</span><span class="o">,</span> <span class="mi">10</span><span class="o">)</span> + <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span></code></pre></div> + +<h2 id="dissecting-flinks-windowing-mechanics">Dissecting Flinkâs windowing mechanics</h2> + +<p>Flinkâs built-in time and count windows cover a wide range of common window use cases. However, there are of course applications that require custom windowing logic that cannot be addressed by Flinkâs built-in windows. In order to support also applications that need very specific windowing semantics, the DataStream API exposes interfaces for the internals of its windowing mechanics. These interfaces give very fine-grained control about the way that windows are built and evaluated.</p> + +<p>The following figure depicts Flinkâs windowing mechanism and introduces the components being involved.</p> + +<center> +<img src="/img/blog/window-intro/window-mechanics.png" style="width:90%;margin:15px" /> +</center> + +<p>Elements that arrive at a window operator are handed to a <code>WindowAssigner</code>. The WindowAssigner assigns elements to one or more windows, possibly creating new windows. A <code>Window</code> itself is just an identifier for a list of elements and may provide some optional meta information, such as begin and end time in case of a <code>TimeWindow</code>. Note that an element can be added to multiple windows, which also means that multiple windows can exist at the same time.</p> + +<p>Each window owns a <code>Trigger</code> that decides when the window is evaluated or purged. The trigger is called for each element that is inserted into the window and when a previously registered timer times out. On each event, a trigger can decide to fire (i.e., evaluate), purge (remove the window and discard its content), or fire and then purge the window. A trigger that just fires evaluates the window and keeps it as it is, i.e., all elements remain in the window and are evaluated again when the triggers fires the next time. A window can be evaluated several times and exists until it is purged. Note that a window consumes memory until it is purged.</p> + +<p>When a Trigger fires, the list of window elements can be given to an optional <code>Evictor</code>. The evictor can iterate through the list and decide to cut off some elements from the start of the list, i.e., remove some of the elements that entered the window first. The remaining elements are given to an evaluation function. If no Evictor was defined, the Trigger hands all the window elements directly to the evaluation function.</p> + +<p>The evaluation function receives the elements of a window (possibly filtered by an Evictor) and computes one or more result elements for the window. The DataStream API accepts different types of evaluation functions, including predefined aggregation functions such as <code>sum()</code>, <code>min()</code>, <code>max()</code>, as well as a <code>ReduceFunction</code>, <code>FoldFunction</code>, or <code>WindowFunction</code>. A WindowFunction is the most generic evaluation function and receives the window object (i.e, the meta data of the window), the list of window elements, and the window key (in case of a keyed window) as parameters.</p> + +<p>These are the components that constitute Flinkâs windowing mechanics. We now show step-by-step how to implement custom windowing logic with the DataStream API. We start with a stream of type <code>DataStream[IN]</code> and key it using a key selector function that extracts a key of type <code>KEY</code> to obtain a <code>KeyedStream[IN, KEY]</code>.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="k">val</span> <span class="n">input</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[</span><span class="kt">IN</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> + +<span class="c1">// created a keyed stream using a key selector function</span> +<span class="k">val</span> <span class="n">keyed</span><span class="k">:</span> <span class="kt">KeyedStream</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">KEY</span><span class="o">]</span> <span class="k">=</span> <span class="n">input</span> + <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="n">myKeySel</span><span class="k">:</span> <span class="o">(</span><span class="kt">IN</span><span class="o">)</span> <span class="o">=></span> <span class="nc">KEY</span><span class="o">)</span></code></pre></div> + +<p>We apply a <code>WindowAssigner[IN, WINDOW]</code> that creates windows of type <code>WINDOW</code> resulting in a <code>WindowedStream[IN, KEY, WINDOW]</code>. In addition, a <code>WindowAssigner</code> also provides a default <code>Trigger</code> implementation.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// create windowed stream using a WindowAssigner</span> +<span class="k">var</span> <span class="n">windowed</span><span class="k">:</span> <span class="kt">WindowedStream</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">KEY</span>, <span class="kt">WINDOW</span><span class="o">]</span> <span class="k">=</span> <span class="n">keyed</span> + <span class="o">.</span><span class="n">window</span><span class="o">(</span><span class="n">myAssigner</span><span class="k">:</span> <span class="kt">WindowAssigner</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> + +<p>We can explicitly specify a <code>Trigger</code> to overwrite the default <code>Trigger</code> provided by the <code>WindowAssigner</code>. Note that specifying a triggers does not add an additional trigger condition but replaces the current trigger.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// override the default trigger of the WindowAssigner</span> +<span class="n">windowed</span> <span class="k">=</span> <span class="n">windowed</span> + <span class="o">.</span><span class="n">trigger</span><span class="o">(</span><span class="n">myTrigger</span><span class="k">:</span> <span class="kt">Trigger</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> + +<p>We may want to specify an optional <code>Evictor</code> as follows.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// specify an optional evictor</span> +<span class="n">windowed</span> <span class="k">=</span> <span class="n">windowed</span> + <span class="o">.</span><span class="n">evictor</span><span class="o">(</span><span class="n">myEvictor</span><span class="k">:</span> <span class="kt">Evictor</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> + +<p>Finally, we apply a <code>WindowFunction</code> that returns elements of type <code>OUT</code> to obtain a <code>DataStream[OUT]</code>.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="c1">// apply window function to windowed stream</span> +<span class="k">val</span> <span class="n">output</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[</span><span class="kt">OUT</span><span class="o">]</span> <span class="k">=</span> <span class="n">windowed</span> + <span class="o">.</span><span class="n">apply</span><span class="o">(</span><span class="n">myWinFunc</span><span class="k">:</span> <span class="kt">WindowFunction</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">OUT</span>, <span class="kt">KEY</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> + +<p>With Flinkâs internal windowing mechanics and its exposure through the DataStream API it is possible to implement very custom windowing logic such as session windows or windows that emit early results if the values exceed a certain threshold.</p> + +<h2 id="conclusion">Conclusion</h2> + +<p>Support for various types of windows over continuous data streams is a must-have for modern stream processors. Apache Flink is a stream processor with a very strong feature set, including a very flexible mechanism to build and evaluate windows over continuous data streams. Flink provides pre-defined window operators for common uses cases as well as a toolbox that allows to define very custom windowing logic. The Flink community will add more pre-defined window operators as we learn the requirements from our users.</p> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/12/11/storm-compatibility.html ---------------------------------------------------------------------- diff --git a/content/news/2015/12/11/storm-compatibility.html b/content/news/2015/12/11/storm-compatibility.html new file mode 100644 index 0000000..c50b031 --- /dev/null +++ b/content/news/2015/12/11/storm-compatibility.html @@ -0,0 +1,339 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Storm Compatibility in Apache Flink: How to run existing Storm topologies on Flink</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Storm Compatibility in Apache Flink: How to run existing Storm topologies on Flink</h1> + + <article> + <p>11 Dec 2015 by Matthias J. Sax (<a href="https://twitter.com/MatthiasJSax";>@MatthiasJSax</a>)</p> + +<p><a href="https://storm.apache.org";>Apache Storm</a> was one of the first distributed and scalable stream processing systems available in the open source space offering (near) real-time tuple-by-tuple processing semantics. +Initially released by the developers at Backtype in 2011 under the Eclipse open-source license, it became popular very quickly. +Only shortly afterwards, Twitter acquired Backtype. +Since then, Storm has been growing in popularity, is used in production at many big companies, and is the de-facto industry standard for big data stream processing. +In 2013, Storm entered the Apache incubator program, followed by its graduation to top-level in 2014.</p> + +<p>Apache Flink is a stream processing engine that improves upon older technologies like Storm in several dimensions, +including <a href="https://ci.apache.org/projects/flink/flink-docs-master/internals/stream_checkpointing.html";>strong consistency guarantees</a> (âexactly onceâ), +a higher level <a href="https://ci.apache.org/projects/flink/flink-docs-master/apis/streaming_guide.html";>DataStream API</a>, +support for <a href="http://flink.apache.org/news/2015/12/04/Introducing-windows.html";>event time and a rich windowing system</a>, +as well as <a href="https://data-artisans.com/high-throughput-low-latency-and-exactly-once-stream-processing-with-apache-flink/";>superior throughput with competitive low latency</a>.</p> + +<p>While Flink offers several technical benefits over Storm, an existing investment on a codebase of applications developed for Storm often makes it difficult to switch engines. +For these reasons, as part of the Flink 0.10 release, Flink ships with a Storm compatibility package that allows users to:</p> + +<ul> + <li>Run <strong>unmodified</strong> Storm topologies using Apache Flink benefiting from superior performance.</li> + <li><strong>Embed</strong> Storm code (spouts and bolts) as operators inside Flink DataStream programs.</li> +</ul> + +<p>Only minor code changes are required in order to submit the program to Flink instead of Storm. +This minimizes the work for developers to run existing Storm topologies while leveraging Apache Flinkâs fast and robust execution engine.</p> + +<p>We note that the Storm compatibility package is continuously improving and does not cover the full spectrum of Stormâs API. +However, it is powerful enough to cover many use cases.</p> + +<h2 id="executing-storm-topologies-with-flink">Executing Storm topologies with Flink</h2> + +<center> +<img src="/img/blog/flink-storm.png" style="height:200px;margin:15px" /> +</center> + +<p>The easiest way to use the Storm compatibility package is by executing a whole Storm topology in Flink. +For this, you only need to replace the dependency <code>storm-core</code> by <code>flink-storm</code> in your Storm project and <strong>change two lines of code</strong> in your original Storm program.</p> + +<p>The following example shows a simple Storm-Word-Count-Program that can be executed in Flink. +First, the program is assembled the Storm way without any code change to Spouts, Bolts, or the topology itself.</p> + +<div class="highlight"><pre><code class="language-java"><span class="c1">// assemble topology, the Storm way</span> +<span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TopologyBuilder</span><span class="o">();</span> +<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"source"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">StormFileSpout</span><span class="o">(</span><span class="n">inputFilePath</span><span class="o">));</span> +<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"tokenizer"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">StormBoltTokenizer</span><span class="o">())</span> + <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"source"</span><span class="o">);</span> +<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"counter"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">StormBoltCounter</span><span class="o">())</span> + <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"tokenizer"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span> +<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"sink"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">StormBoltFileSink</span><span class="o">(</span><span class="n">outputFilePath</span><span class="o">))</span> + <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"counter"</span><span class="o">);</span></code></pre></div> + +<p>In order to execute the topology, we need to translate it to a <code>FlinkTopology</code> and submit it to a local or remote Flink cluster, very similar to submitting the application to a Storm cluster.<sup><a href="#fn1" id="ref1">1</a></sup></p> + +<div class="highlight"><pre><code class="language-java"><span class="c1">// transform Storm topology to Flink program</span> +<span class="c1">// replaces: StormTopology topology = builder.createTopology();</span> +<span class="n">FlinkTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="n">FlinkTopology</span><span class="o">.</span><span class="na">createTopology</span><span class="o">(</span><span class="n">builder</span><span class="o">);</span> + +<span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span> +<span class="k">if</span><span class="o">(</span><span class="n">runLocal</span><span class="o">)</span> <span class="o">{</span> + <span class="c1">// use FlinkLocalCluster instead of LocalCluster</span> + <span class="n">FlinkLocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="n">FlinkLocalCluster</span><span class="o">.</span><span class="na">getLocalCluster</span><span class="o">();</span> + <span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span> +<span class="o">}</span> <span class="k">else</span> <span class="o">{</span> + <span class="c1">// use FlinkSubmitter instead of StormSubmitter</span> + <span class="n">FlinkSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span> +<span class="o">}</span></code></pre></div> + +<p>As a shorter Flink-style alternative that replaces the Storm-style submission code, you can also use context-based job execution:</p> + +<div class="highlight"><pre><code class="language-java"><span class="c1">// transform Storm topology to Flink program (as above)</span> +<span class="n">FlinkTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="n">FlinkTopology</span><span class="o">.</span><span class="na">createTopology</span><span class="o">(</span><span class="n">builder</span><span class="o">);</span> + +<span class="c1">// executes locally by default or remotely if submitted with Flink's command-line client</span> +<span class="n">topology</span><span class="o">.</span><span class="na">execute</span><span class="o">()</span></code></pre></div> + +<p>After the code is packaged in a jar file (e.g., <code>StormWordCount.jar</code>), it can be easily submitted to Flink via</p> + +<div class="highlight"><pre><code>bin/flink run StormWordCount.jar +</code></pre></div> + +<p>The used Spouts and Bolts as well as the topology assemble code is not changed at all! +Only the translation and submission step have to be changed to the Storm-API compatible Flink pendants. +This allows for minimal code changes and easy adaption to Flink.</p> + +<h3 id="embedding-spouts-and-bolts-in-flink-programs">Embedding Spouts and Bolts in Flink programs</h3> + +<p>It is also possible to use Spouts and Bolts within a regular Flink DataStream program. +The compatibility package provides wrapper classes for Spouts and Bolts which are implemented as a Flink <code>SourceFunction</code> and <code>StreamOperator</code> respectively. +Those wrappers automatically translate incoming Flink POJO and <code>TupleXX</code> records into Stormâs <code>Tuple</code> type and emitted <code>Values</code> back into either POJOs or <code>TupleXX</code> types for further processing by Flink operators. +As Storm is type agnostic, it is required to specify the output type of embedded Spouts/Bolts manually to get a fully typed Flink streaming program.</p> + +<div class="highlight"><pre><code class="language-java"><span class="c1">// use regular Flink streaming environment</span> +<span class="n">StreamExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">StreamExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> + +<span class="c1">// use Spout as source</span> +<span class="n">DataStream</span><span class="o"><</span><span class="n">Tuple1</span><span class="o"><</span><span class="n">String</span><span class="o">>></span> <span class="n">source</span> <span class="o">=</span> + <span class="n">env</span><span class="o">.</span><span class="na">addSource</span><span class="o">(</span><span class="c1">// Flink provided wrapper including original Spout</span> + <span class="k">new</span> <span class="n">SpoutWrapper</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="k">new</span> <span class="nf">FileSpout</span><span class="o">(</span><span class="n">localFilePath</span><span class="o">)),</span> + <span class="c1">// specify output type manually</span> + <span class="n">TypeExtractor</span><span class="o">.</span><span class="na">getForObject</span><span class="o">(</span><span class="k">new</span> <span class="n">Tuple1</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="s">""</span><span class="o">)));</span> +<span class="c1">// FileSpout cannot be parallelized</span> +<span class="n">DataStream</span><span class="o"><</span><span class="n">Tuple1</span><span class="o"><</span><span class="n">String</span><span class="o">>></span> <span class="n">text</span> <span class="o">=</span> <span class="n">source</span><span class="o">.</span><span class="na">setParallelism</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span> + +<span class="c1">// further processing with Flink</span> +<span class="n">DataStream</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span><span class="n">Integer</span><span class="o">></span> <span class="n">tokens</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()).</span><span class="na">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span> + +<span class="c1">// use Bolt for counting</span> +<span class="n">DataStream</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span><span class="n">Integer</span><span class="o">></span> <span class="n">counts</span> <span class="o">=</span> + <span class="n">tokens</span><span class="o">.</span><span class="na">transform</span><span class="o">(</span><span class="s">"Counter"</span><span class="o">,</span> + <span class="c1">// specify output type manually</span> + <span class="n">TypeExtractor</span><span class="o">.</span><span class="na">getForObject</span><span class="o">(</span><span class="k">new</span> <span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span><span class="n">Integer</span><span class="o">>(</span><span class="s">""</span><span class="o">,</span><span class="mi">0</span><span class="o">))</span> + <span class="c1">// Flink provided wrapper including original Bolt</span> + <span class="k">new</span> <span class="n">BoltWrapper</span><span class="o"><</span><span class="n">String</span><span class="o">,</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span><span class="n">Integer</span><span class="o">>>(</span><span class="k">new</span> <span class="nf">BoltCounter</span><span class="o">()));</span> + +<span class="c1">// write result to file via Flink sink</span> +<span class="n">counts</span><span class="o">.</span><span class="na">writeAsText</span><span class="o">(</span><span class="n">outputPath</span><span class="o">);</span> + +<span class="c1">// start Flink job</span> +<span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"WordCount with Spout source and Bolt counter"</span><span class="o">);</span></code></pre></div> + +<p>Although some boilerplate code is needed (we plan to address this soon!), the actual embedded Spout and Bolt code can be used unmodified. +We also note that the resulting program is fully typed, and type errors will be found by Flinkâs type extractor even if the original Spouts and Bolts are not.</p> + +<h2 id="outlook">Outlook</h2> + +<p>The Storm compatibility package is currently in beta and undergoes continuous development. +We are currently working on providing consistency guarantees for stateful Bolts. +Furthermore, we want to provide a better API integration for embedded Spouts and Bolts by providing a âStormExecutionEnvironmentâ as a special extension of Flinkâs <code>StreamExecutionEnvironment</code>. +We are also investigating the integration of Stormâs higher-level programming API Trident.</p> + +<h2 id="summary">Summary</h2> + +<p>Flinkâs compatibility package for Storm allows using unmodified Spouts and Bolts within Flink. +This enables you to even embed third-party Spouts and Bolts where the source code is not available. +While you can embed Spouts/Bolts in a Flink program and mix-and-match them with Flink operators, running whole topologies is the easiest way to get started and can be achieved with almost no code changes.</p> + +<p>If you want to try out Flinkâs Storm compatibility package checkout our <a href="https://ci.apache.org/projects/flink/flink-docs-master/apis/streaming/storm_compatibility.html";>Documentation</a>.</p> + +<hr /> + +<p><sup id="fn1">1. We confess, there are three lines changed compared to a Storm project <img class="emoji" style="width:16px;height:16px;align:absmiddle" src="/img/blog/smirk.png" />âbecause the example covers local <em>and</em> remote execution. <a href="#ref1" title="Back to text.">â©</a></sup></p> + + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/12/18/a-year-in-review.html ---------------------------------------------------------------------- diff --git a/content/news/2015/12/18/a-year-in-review.html b/content/news/2015/12/18/a-year-in-review.html new file mode 100644 index 0000000..c65ddc8 --- /dev/null +++ b/content/news/2015/12/18/a-year-in-review.html @@ -0,0 +1,416 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Flink 2015: A year in review, and a lookout to 2016</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Flink 2015: A year in review, and a lookout to 2016</h1> + + <article> + <p>18 Dec 2015 by Robert Metzger (<a href="https://twitter.com/rmetzger_";>@rmetzger_</a>)</p> + +<p>With 2015 ending, we thought that this would be good time to reflect +on the amazing work done by the Flink community over this past year, +and how much this community has grown.</p> + +<p>Overall, we have seen Flink grow in terms of functionality from an +engine to one of the most complete open-source stream processing +frameworks available. The community grew from a relatively small and +geographically focused team, to a truly global, and one of the largest +big data communities in the the Apache Software Foundation.</p> + +<p>We will also look at some interesting stats, including that the +busiest days for Flink are Mondays (who would have thought :-).</p> + +<h1 id="community-growth">Community growth</h1> + +<p>Let us start with some simple statistics from <a href="https://github.com/apache/flink";>Flinkâs +github repository</a>. During 2015, the +Flink community <strong>doubled</strong> in size, from about 75 contributors to +over 150. Forks of the repository more than <strong>tripled</strong> from 160 in +February 2015 to 544 in December 2015, and the number of stars of the +repository almost tripled from 289 to 813.</p> + +<center> +<img src="/img/blog/community-growth.png" style="height:400px;margin:15px" /> +</center> + +<p>Although Flink started out geographically in Berlin, Germany, the +community is by now spread all around the globe, with many +contributors from North America, Europe, and Asia. A simple search at +meetup.com for groups that mention Flink as a focus area reveals <a href="http://apache-flink.meetup.com/";>16 +meetups around the globe</a>:</p> + +<center> +<img src="/img/blog/meetup-map.png" style="height:400px;margin:15px" /> +</center> + +<h1 id="flink-forward-2015">Flink Forward 2015</h1> + +<p>One of the highlights of the year for Flink was undoubtedly the <a href="http://2015.flink-forward.org/";>Flink +Forward</a> conference, the first conference +on Apache Flink that was held in October in Berlin. More than 250 +participants (roughly half based outside Germany where the conference +was held) attended more than 33 technical talks from organizations +including Google, MongoDB, Bouygues Telecom, NFLabs, Euranova, RedHat, +IBM, Huawei, Intel, Ericsson, Capital One, Zalando, Amadeus, the Otto +Group, and ResearchGate. If you have not yet watched their talks, +check out the <a href="http://2015.flink-forward.org/?post_type=day";>slides</a> and +<a href="https://www.youtube.com/playlist?list=PLDX4T_cnKjD31JeWR1aMOi9LXPRQ6nyHO";>videos</a> +from Flink Forward.</p> + +<center> +<img src="/img/blog/ff-speakers.png" style="height:400px;margin:15px" /> +</center> + +<h1 id="media-coverage">Media coverage</h1> + +<p>And of course, interest in Flink was picked up by the tech +media. During 2015, articles about Flink appeared in +<a href="http://www.infoq.com/Apache-Flink/news/";>InfoQ</a>, +<a href="http://www.zdnet.com/article/five-open-source-big-data-projects-to-watch/";>ZDNet</a>, +<a href="http://www.datanami.com/tag/apache-flink/";>Datanami</a>, +<a href="http://www.infoworld.com/article/2919602/hadoop/flink-hadoops-new-contender-for-mapreduce-spark.html";>Infoworld</a> +(including being one of the <a href="http://www.infoworld.com/article/2982429/open-source-tools/bossie-awards-2015-the-best-open-source-big-data-tools.html";>best open source big data tools of +2015</a>), +the <a href="http://blogs.gartner.com/nick-heudecker/apache-flink-offers-a-challenge-to-spark/";>Gartner +blog</a>, +<a href="http://dataconomy.com/tag/apache-flink/";>Dataconomy</a>, +<a href="http://sdtimes.com/tag/apache-flink/";>SDTimes</a>, the <a href="https://www.mapr.com/blog/apache-flink-new-way-handle-streaming-data";>MapR +blog</a>, +<a href="http://www.kdnuggets.com/2015/08/apache-flink-stream-processing.html";>KDnuggets</a>, +and +<a href="http://www.hadoopsphere.com/2015/02/distributed-data-processing-with-apache.html";>HadoopSphere</a>.</p> + +<center> +<img src="/img/blog/appeared-in.png" style="height:400px;margin:15px" /> +</center> + +<p>It is interesting to see that Hadoop Summit EMEA 2016 had a whopping +number of 17 (!) talks submitted that are mentioning Flink in their +title and abstract:</p> + +<center> +<img src="/img/blog/hadoop-summit.png" style="height:400px;margin:15px" /> +</center> + +<h1 id="fun-with-stats-when-do-committers-commit">Fun with stats: when do committers commit?</h1> + +<p>To get some deeper insight on what is happening in the Flink +community, let us do some analytics on the git log of the project :-) +The easiest thing we can do is count the number of commits at the +repository in 2015. Running</p> + +<div class="highlight"><pre><code>git log --pretty=oneline --after=1/1/2015 | wc -l +</code></pre></div> + +<p>on the Flink repository yields a total of <strong>2203 commits</strong> in 2015.</p> + +<p>To dig deeper, we will use an open source tool called gitstats that +will give us some interesting statistics on the committer +behavior. You can create these also yourself and see many more by +following four easy steps:</p> + +<ol> + <li>Download gitstats from the <a href="http://gitstats.sourceforge.net/";>project homepage</a>.. E.g., on OS X with homebrew, type</li> +</ol> + +<div class="highlight"><pre><code>brew install --HEAD homebrew/head-only/gitstats +</code></pre></div> + +<ol> + <li>Clone the Apache Flink git repository:</li> +</ol> + +<div class="highlight"><pre><code>git clone g...@github.com:apache/flink.git +</code></pre></div> + +<ol> + <li>Generate the statistics</li> +</ol> + +<div class="highlight"><pre><code>gitstats flink/ flink-stats/ +</code></pre></div> + +<ol> + <li>View all the statistics as an html page using your favorite browser (e.g., chrome):</li> +</ol> + +<div class="highlight"><pre><code>chrome flink-stats/index.html +</code></pre></div> + +<p>First, we can see a steady growth of lines of code in Flink since the +initial Apache incubator project. During 2015, the codebase almost +<strong>doubled</strong> from 500,000 LOC to 900,000 LOC.</p> + +<center> +<img src="/img/blog/code-growth.png" style="height:400px;margin:15px" /> +</center> + +<p>It is interesting to see when committers commit. For Flink, Monday +afternoons are by far the most popular times to commit to the +repository:</p> + +<center> +<img src="/img/blog/commit-stats.png" style="height:400px;margin:15px" /> +</center> + +<h1 id="feature-timeline">Feature timeline</h1> + +<p>So, what were the major features added to Flink and the Flink +ecosystem during 2015? Here is a (non-exhaustive) chronological list:</p> + +<center> +<img src="/img/blog/feature-timeline.png" style="height:400px;margin:15px" /> +</center> + +<h1 id="roadmap-for-2016">Roadmap for 2016</h1> + +<p>With 2015 coming to a close, the Flink community has already started +discussing Flinkâs roadmap for the future. Some highlights +are:</p> + +<ul> + <li> + <p><strong>Runtime scaling of streaming jobs:</strong> streaming jobs are running + forever, and need to react to a changing environment. Runtime + scaling means dynamically increasing and decreasing the + parallelism of a job to sustain certain SLAs, or react to changing + input throughput.</p> + </li> + <li> + <p><strong>SQL queries for static data sets and streams:</strong> building on top of + Flinkâs Table API, users should be able to write SQL + queries for static data sets, as well as SQL queries on data + streams that continuously produce new results.</p> + </li> + <li> + <p><strong>Streaming operators backed by managed memory:</strong> currently, + streaming operators like user-defined state and windows are backed + by JVM heap objects. Moving those to Flink managed memory will add + the ability to spill to disk, GC efficiency, as well as better + control over memory utilization.</p> + </li> + <li> + <p><strong>Library for detecting temporal event patterns:</strong> a common use case + for stream processing is detecting patterns in an event stream + with timestamps. Flink makes this possible with its support for + event time, so many of these operators can be surfaced in the form + of a library.</p> + </li> + <li> + <p><strong>Support for Apache Mesos, and resource-dynamic YARN support:</strong> + support for both Mesos and YARN, including dynamic allocation and + release of resource for more resource elasticity (for both batch + and stream processing).</p> + </li> + <li> + <p><strong>Security:</strong> encrypt both the messages exchanged between + TaskManagers and JobManager, as well as the connections for data + exchange between workers.</p> + </li> + <li> + <p><strong>More streaming connectors, more runtime metrics, and continuous + DataStream API enhancements:</strong> add support for more sources and + sinks (e.g., Amazon Kinesis, Cassandra, Flume, etc), expose more + metrics to the user, and provide continuous improvements to the + DataStream API.</p> + </li> +</ul> + +<p>If you are interested in these features, we highly encourage you to +take a look at the <a href="https://docs.google.com/document/d/1ExmtVpeVVT3TIhO1JoBpC5JKXm-778DAD7eqw5GANwE/edit";>current +draft</a>, +and <a href="https://mail-archives.apache.org/mod_mbox/flink-dev/201512.mbox/browser";>join the +discussion</a> +on the Flink mailing lists.</p> + + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2016/02/11/release-0.10.2.html ---------------------------------------------------------------------- diff --git a/content/news/2016/02/11/release-0.10.2.html b/content/news/2016/02/11/release-0.10.2.html new file mode 100644 index 0000000..83299bd --- /dev/null +++ b/content/news/2016/02/11/release-0.10.2.html @@ -0,0 +1,229 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Flink 0.10.2 Released</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Flink 0.10.2 Released</h1> + + <article> + <p>11 Feb 2016</p> + +<p>Today, the Flink community released Flink version <strong>0.10.2</strong>, the second bugfix release of the 0.10 series.</p> + +<p>We <strong>recommend all users updating to this release</strong> by bumping the version of your Flink dependencies to <code>0.10.2</code> and updating the binaries on the server.</p> + +<h2 id="issues-fixed">Issues fixed</h2> + +<ul> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3242";>FLINK-3242</a>: Adjust StateBackendITCase for 0.10 signatures of state backends</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3236";>FLINK-3236</a>: Flink user code classloader as parent classloader from Flink core classes</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-2962";>FLINK-2962</a>: Cluster startup script refers to unused variable</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3151";>FLINK-3151</a>: Downgrade to Netty version 4.0.27.Final</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3224";>FLINK-3224</a>: Call setInputType() on output formats that implement InputTypeConfigurable</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3218";>FLINK-3218</a>: Fix overriding of user parameters when merging Hadoop configurations</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3189";>FLINK-3189</a>: Fix argument parsing of CLI client INFO action</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3176";>FLINK-3176</a>: Improve documentation for window apply</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3185";>FLINK-3185</a>: Log error on failure during recovery</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3185";>FLINK-3185</a>: Donât swallow test failure Exception</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3147";>FLINK-3147</a>: Expose HadoopOutputFormatBase fields as protected</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3145";>FLINK-3145</a>: Pin Kryo version of transitive dependencies</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3143";>FLINK-3143</a>: Update Closure Cleanerâs ASM references to ASM5</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3136";>FLINK-3136</a>: Fix shaded imports in ClosureCleaner.scala</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3108";>FLINK-3108</a>: JoinOperatorâs with() calls the wrong TypeExtractor method</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3125";>FLINK-3125</a>: Web server starts also when JobManager log files cannot be accessed.</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3080";>FLINK-3080</a>: Relax restrictions of DataStream.union()</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3081";>FLINK-3081</a>: Properly stop periodic Kafka committer</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3082";>FLINK-3082</a>: Fixed confusing error about an interface that no longer exists</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3067";>FLINK-3067</a>: Enforce zkclient 0.7 for Kafka</li> + <li><a href="https://issues.apache.org/jira/browse/FLINK-3020";>FLINK-3020</a>: Set number of task slots to maximum parallelism in local execution</li> +</ul> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html>
