http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2014/11/18/hadoop-compatibility.html ---------------------------------------------------------------------- diff --git a/content/news/2014/11/18/hadoop-compatibility.html b/content/news/2014/11/18/hadoop-compatibility.html new file mode 100644 index 0000000..7cedbe2 --- /dev/null +++ b/content/news/2014/11/18/hadoop-compatibility.html @@ -0,0 +1,279 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Hadoop Compatibility in Flink</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Hadoop Compatibility in Flink</h1> + + <article> + <p>18 Nov 2014 by Fabian Hüske (<a href="https://twitter.com/fhueske";>@fhueske</a>)</p> + +<p><a href="http://hadoop.apache.org";>Apache Hadoop</a> is an industry standard for scalable analytical data processing. Many data analysis applications have been implemented as Hadoop MapReduce jobs and run in clusters around the world. Apache Flink can be an alternative to MapReduce and improves it in many dimensions. Among other features, Flink provides much better performance and offers APIs in Java and Scala, which are very easy to use. Similar to Hadoop, Flinkâs APIs provide interfaces for Mapper and Reducer functions, as well as Input- and OutputFormats along with many more operators. While being conceptually equivalent, Hadoopâs MapReduce and Flinkâs interfaces for these functions are unfortunately not source compatible.</p> + +<h2 id="flinks-hadoop-compatibility-package">Flinkâs Hadoop Compatibility Package</h2> + +<center> +<img src="/img/blog/hcompat-logos.png" style="width:30%;margin:15px" /> +</center> + +<p>To close this gap, Flink provides a Hadoop Compatibility package to wrap functions implemented against Hadoopâs MapReduce interfaces and embed them in Flink programs. This package was developed as part of a <a href="https://developers.google.com/open-source/soc/";>Google Summer of Code</a> 2014 project.</p> + +<p>With the Hadoop Compatibility package, you can reuse all your Hadoop</p> + +<ul> + <li><code>InputFormats</code> (mapred and mapreduce APIs)</li> + <li><code>OutputFormats</code> (mapred and mapreduce APIs)</li> + <li><code>Mappers</code> (mapred API)</li> + <li><code>Reducers</code> (mapred API)</li> +</ul> + +<p>in Flink programs without changing a line of code. Moreover, Flink also natively supports all Hadoop data types (<code>Writables</code> and <code>WritableComparable</code>).</p> + +<p>The following code snippet shows a simple Flink WordCount program that solely uses Hadoop data types, InputFormat, OutputFormat, Mapper, and Reducer functions.</p> + +<div class="highlight"><pre><code class="language-java"><span class="c1">// Definition of Hadoop Mapper function</span> +<span class="kd">public</span> <span class="kd">class</span> <span class="nc">Tokenizer</span> <span class="kd">implements</span> <span class="n">Mapper</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">></span> <span class="o">{</span> <span class="o">...</span> <span class="o">}</span> +<span class="c1">// Definition of Hadoop Reducer function</span> +<span class="kd">public</span> <span class="kd">class</span> <span class="nc">Counter</span> <span class="kd">implements</span> <span class="n">Reducer</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">></span> <span class="o">{</span> <span class="o">...</span> <span class="o">}</span> + +<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span> + <span class="kd">final</span> <span class="n">String</span> <span class="n">inputPath</span> <span class="o">=</span> <span class="n">args</span><span class="o">[</span><span class="mi">0</span><span class="o">];</span> + <span class="kd">final</span> <span class="n">String</span> <span class="n">outputPath</span> <span class="o">=</span> <span class="n">args</span><span class="o">[</span><span class="mi">1</span><span class="o">];</span> + + <span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> + + <span class="c1">// Setup Hadoopâs TextInputFormat</span> + <span class="n">HadoopInputFormat</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">></span> <span class="n">hadoopInputFormat</span> <span class="o">=</span> + <span class="k">new</span> <span class="n">HadoopInputFormat</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">>(</span> + <span class="k">new</span> <span class="nf">TextInputFormat</span><span class="o">(),</span> <span class="n">LongWritable</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">Text</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="k">new</span> <span class="nf">JobConf</span><span class="o">());</span> + <span class="n">TextInputFormat</span><span class="o">.</span><span class="na">addInputPath</span><span class="o">(</span><span class="n">hadoopInputFormat</span><span class="o">.</span><span class="na">getJobConf</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">inputPath</span><span class="o">));</span> + + <span class="c1">// Read a DataSet with the Hadoop InputFormat</span> + <span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">>></span> <span class="n">text</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">createInput</span><span class="o">(</span><span class="n">hadoopInputFormat</span><span class="o">);</span> + <span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>></span> <span class="n">words</span> <span class="o">=</span> <span class="n">text</span> + <span class="c1">// Wrap Tokenizer Mapper function</span> + <span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopMapFunction</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span><span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()))</span> + <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="c1">// Wrap Counter Reducer function (used as Reducer and Combiner)</span> + <span class="o">.</span><span class="na">reduceGroup</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopReduceCombineFunction</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> + <span class="k">new</span> <span class="nf">Counter</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Counter</span><span class="o">()));</span> + + <span class="c1">// Setup Hadoopâs TextOutputFormat</span> + <span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">></span> <span class="n">hadoopOutputFormat</span> <span class="o">=</span> + <span class="k">new</span> <span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> + <span class="k">new</span> <span class="n">TextOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(),</span> <span class="k">new</span> <span class="nf">JobConf</span><span class="o">());</span> + <span class="n">hadoopOutputFormat</span><span class="o">.</span><span class="na">getJobConf</span><span class="o">().</span><span class="na">set</span><span class="o">(</span><span class="s">"mapred.textoutputformat.separator"</span><span class="o">,</span> <span class="s">" "</span><span class="o">);</span> + <span class="n">TextOutputFormat</span><span class="o">.</span><span class="na">setOutputPath</span><span class="o">(</span><span class="n">hadoopOutputFormat</span><span class="o">.</span><span class="na">getJobConf</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">outputPath</span><span class="o">));</span> + + <span class="c1">// Output & Execute</span> + <span class="n">words</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="n">hadoopOutputFormat</span><span class="o">);</span> + <span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"Hadoop Compat WordCount"</span><span class="o">);</span> +<span class="o">}</span></code></pre></div> + +<p>As you can see, Flink represents Hadoop key-value pairs as <code>Tuple2<key, value></code> tuples. Note, that the program uses Flinkâs <code>groupBy()</code> transformation to group data on the key field (field 0 of the <code>Tuple2<key, value></code>) before it is given to the Reducer function. At the moment, the compatibility package does not evaluate custom Hadoop partitioners, sorting comparators, or grouping comparators.</p> + +<p>Hadoop functions can be used at any position within a Flink program and of course also be mixed with native Flink functions. This means that instead of assembling a workflow of Hadoop jobs in an external driver method or using a workflow scheduler such as <a href="http://oozie.apache.org";>Apache Oozie</a>, you can implement an arbitrary complex Flink program consisting of multiple Hadoop Input- and OutputFormats, Mapper and Reducer functions. When executing such a Flink program, data will be pipelined between your Hadoop functions and will not be written to HDFS just for the purpose of data exchange.</p> + +<center> +<img src="/img/blog/hcompat-flow.png" style="width:100%;margin:15px" /> +</center> + +<h2 id="what-comes-next">What comes next?</h2> + +<p>While the Hadoop compatibility package is already very useful, we are currently working on a dedicated Hadoop Job operation to embed and execute Hadoop jobs as a whole in Flink programs, including their custom partitioning, sorting, and grouping code. With this feature, you will be able to chain multiple Hadoop jobs, mix them with Flink functions, and other operations such as <a href="http://ci.apache.org/projects/flink/flink-docs-release-0.7/spargel_guide.html";>Spargel</a> operations (Pregel/Giraph-style jobs).</p> + +<h2 id="summary">Summary</h2> + +<p>Flink lets you reuse a lot of the code you wrote for Hadoop MapReduce, including all data types, all Input- and OutputFormats, and Mapper and Reducers of the mapred-API. Hadoop functions can be used within Flink programs and mixed with all other Flink functions. Due to Flinkâs pipelined execution, Hadoop functions can arbitrarily be assembled without data exchange via HDFS. Moreover, the Flink community is currently working on a dedicated Hadoop Job operation to supporting the execution of Hadoop jobs as a whole.</p> + +<p>If you want to use Flinkâs Hadoop compatibility package checkout our <a href="https://ci.apache.org/projects/flink/flink-docs-master/apis/batch/hadoop_compatibility.html";>documentation</a>.</p> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/01/06/december-in-flink.html ---------------------------------------------------------------------- diff --git a/content/news/2015/01/06/december-in-flink.html b/content/news/2015/01/06/december-in-flink.html new file mode 100644 index 0000000..b032674 --- /dev/null +++ b/content/news/2015/01/06/december-in-flink.html @@ -0,0 +1,254 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: December 2014 in the Flink community</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>December 2014 in the Flink community</h1> + + <article> + <p>06 Jan 2015</p> + +<p>This is the first blog post of a ânewsletterâ like series where we give a summary of the monthly activity in the Flink community. As the Flink project grows, this can serve as a âtl;drâ for people that are not following the Flink dev and user mailing lists, or those that are simply overwhelmed by the traffic.</p> + +<h3 id="flink-graduation">Flink graduation</h3> + +<p>The biggest news is that the Apache board approved Flink as a top-level Apache project! The Flink team is working closely with the Apache press team for an official announcement, so stay tuned for details!</p> + +<h3 id="new-flink-website">New Flink website</h3> + +<p>The <a href="http://flink.apache.org";>Flink website</a> got a total make-over, both in terms of appearance and content.</p> + +<h3 id="flink-irc-channel">Flink IRC channel</h3> + +<p>A new IRC channel called #flink was created at irc.freenode.org. An easy way to access the IRC channel is through the <a href="http://webchat.freenode.net/";>web client</a>. Feel free to stop by to ask anything or share your ideas about Apache Flink!</p> + +<h3 id="meetups-and-talks">Meetups and Talks</h3> + +<p>Apache Flink was presented in the <a href="http://www.meetup.com/Netherlands-Hadoop-User-Group/events/218635152";>Amsterdam Hadoop User Group</a></p> + +<h2 id="notable-code-contributions">Notable code contributions</h2> + +<p><strong>Note:</strong> Code contributions listed here may not be part of a release or even the current snapshot yet.</p> + +<h3 id="streaming-scala-apihttpsgithubcomapacheincubator-flinkpull275"><a href="https://github.com/apache/incubator-flink/pull/275";>Streaming Scala API</a></h3> + +<p>The Flink Streaming Java API recently got its Scala counterpart. Once merged, Flink Streaming users can use both Scala and Java for their development. The Flink Streaming Scala API is built as a thin layer on top of the Java API, making sure that the APIs are kept easily in sync.</p> + +<h3 id="intermediate-datasetshttpsgithubcomapacheincubator-flinkpull254"><a href="https://github.com/apache/incubator-flink/pull/254";>Intermediate datasets</a></h3> + +<p>This pull request introduces a major change in the Flink runtime. Currently, the Flink runtime is based on the notion of operators that exchange data through channels. With the PR, intermediate data sets that are produced by operators become first-class citizens in the runtime. While this does not have any user-facing impact yet, it lays the groundwork for a slew of future features such as blocking execution, fine-grained fault-tolerance, and more efficient data sharing between cluster and client.</p> + +<h3 id="configurable-execution-modehttpsgithubcomapacheincubator-flinkpull259"><a href="https://github.com/apache/incubator-flink/pull/259";>Configurable execution mode</a></h3> + +<p>This pull request allows the user to change the object-reuse behaviour. Before this pull request, some operations would reuse objects passed to the user function while others would always create new objects. This introduces a system wide switch and changes all operators to either reuse objects or donât reuse objects.</p> + +<h3 id="distributed-coordination-via-akkahttpsgithubcomapacheincubator-flinkpull149"><a href="https://github.com/apache/incubator-flink/pull/149";>Distributed Coordination via Akka</a></h3> + +<p>Another major change is a complete rewrite of the JobManager / TaskManager components in Scala. In addition to that, the old RPC service was replaced by Actors, using the Akka framework.</p> + +<h3 id="sorting-of-very-large-recordshttpsgithubcomapacheincubator-flinkpull249-"><a href="https://github.com/apache/incubator-flink/pull/249";>Sorting of very large records</a></h3> + +<p>Flinkâs internal sort-algorithms were improved to better handle large records (multiple 100s of megabytes or larger). Previously, the system did in some cases hold instances of multiple large records, resulting in high memory consumption and JVM heap thrashing. Through this fix, large records are streamed through the operators, reducing the memory consumption and GC pressure. The system now requires much less memory to support algorithms that work on such large records.</p> + +<h3 id="kryo-serialization-as-the-new-default-fallbackhttpsgithubcomapacheincubator-flinkpull271"><a href="https://github.com/apache/incubator-flink/pull/271";>Kryo Serialization as the new default fallback</a></h3> + +<p>Flinkâs build-in type serialization framework is handles all common types very efficiently. Prior versions uses Avro to serialize types that the built-in framework could not handle. +Flink serialization system improved a lot over time and by now surpasses the capabilities of Avro in many cases. Kryo now serves as the default fallback serialization framework, supporting a much broader range of types.</p> + +<h3 id="hadoop-filesystem-supporthttpsgithubcomapacheincubator-flinkpull268"><a href="https://github.com/apache/incubator-flink/pull/268";>Hadoop FileSystem support</a></h3> + +<p>This change permits users to use all file systems supported by Hadoop with Flink. In practice this means that users can use Flink with Tachyon, Google Cloud Storage (also out of the box Flink YARN support on Google Compute Cloud), FTP and all the other file system implementations for Hadoop.</p> + +<h2 id="heading-to-the-080-release">Heading to the 0.8.0 release</h2> + +<p>The community is working hard together with the Apache infra team to migrate the Flink infrastructure to a top-level project. At the same time, the Flink community is working on the Flink 0.8.0 release which should be out very soon.</p> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/01/21/release-0.8.html ---------------------------------------------------------------------- diff --git a/content/news/2015/01/21/release-0.8.html b/content/news/2015/01/21/release-0.8.html new file mode 100644 index 0000000..de3eee2 --- /dev/null +++ b/content/news/2015/01/21/release-0.8.html @@ -0,0 +1,278 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: Apache Flink 0.8.0 available</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>Apache Flink 0.8.0 available</h1> + + <article> + <p>21 Jan 2015</p> + +<p>We are pleased to announce the availability of Flink 0.8.0. This release includes new user-facing features as well as performance and bug fixes, extends the support for filesystems and introduces the Scala API and flexible windowing semantics for Flink Streaming. A total of 33 people have contributed to this release, a big thanks to all of them!</p> + +<p><a href="http://www.apache.org/dyn/closer.cgi/flink/flink-0.8.0/flink-0.8.0-bin-hadoop2.tgz";>Download Flink 0.8.0</a></p> + +<p><a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315522&version=12328699";>See the release changelog</a></p> + +<h2 id="overview-of-major-new-features">Overview of major new features</h2> + +<ul> + <li> + <p><strong>Extended filesystem support</strong>: The former <code>DistributedFileSystem</code> interface has been generalized to <code>HadoopFileSystem</code> now supporting all sub classes of <code>org.apache.hadoop.fs.FileSystem</code>. This allows users to use all file systems supported by Hadoop with Apache Flink. +<a href="http://ci.apache.org/projects/flink/flink-docs-release-0.8/example_connectors.html";>See connecting to other systems</a></p> + </li> + <li> + <p><strong>Streaming Scala API</strong>: As an alternative to the existing Java API Streaming is now also programmable in Scala. The Java and Scala APIs have now the same syntax and transformations and will be kept from now on in sync in every future release.</p> + </li> + <li> + <p><strong>Streaming windowing semantics</strong>: The new windowing api offers an expressive way to define custom logic for triggering the execution of a stream window and removing elements. The new features include out-of-the-box support for windows based in logical or physical time and data-driven properties on the events themselves among others. <a href="http://ci.apache.org/projects/flink/flink-docs-release-0.8/streaming_guide.html#window-operators";>Read more here</a></p> + </li> + <li> + <p><strong>Mutable and immutable objects in runtime</strong> All Flink versions before 0.8.0 were always passing the same objects to functions written by users. This is a common performance optimization, also used in other systems such as Hadoop. + However, this is error-prone for new users because one has to carefully check that references to the object arenât kept in the user function. Starting from 0.8.0, Flink allows to configure a mode which is disabling that mechanism.</p> + </li> + <li><strong>Performance and usability improvements</strong>: The new Apache Flink 0.8.0 release brings several new features which will significantly improve the performance and the usability of the system. Amongst others, these features include: + <ul> + <li>Improved input split assignment which maximizes computation locality</li> + <li>Smart broadcasting mechanism which minimizes network I/O</li> + <li>Custom partitioners which let the user control how the data is partitioned within the cluster. This helps to prevent data skewness and allows to implement highly efficient algorithms.</li> + <li>coGroup operator now supports group sorting for its inputs</li> + </ul> + </li> + <li> + <p><strong>Kryo is the new fallback serializer</strong>: Apache Flink has a sophisticated type analysis and serialization framework that is able to handle commonly used types very efficiently. + In addition to that, there is a fallback serializer for types which are not supported. Older versions of Flink used the reflective <a href="http://avro.apache.org/";>Avro</a> serializer for that purpose. With this release, Flink is using the powerful <a href="https://github.com/EsotericSoftware/kryo";>Kryo</a> and twitter-chill library for support of types such as Java Collections and Scala specifc types.</p> + </li> + <li> + <p><strong>Hadoop 2.2.0+ is now the default Hadoop dependency</strong>: With Flink 0.8.0 we made the âhadoop2â build profile the default build for Flink. This means that all users using Hadoop 1 (0.2X or 1.2.X versions) have to specify version â0.8.0-hadoop1â in their pom files.</p> + </li> + <li><strong>HBase module updated</strong> The HBase version has been updated to 0.98.6.1. Also, Hbase is now available to the Hadoop1 and Hadoop2 profile of Flink.</li> +</ul> + +<h2 id="contributors">Contributors</h2> + +<ul> + <li>Marton Balassi</li> + <li>Daniel Bali</li> + <li>Carsten Brandt</li> + <li>Moritz Borgmann</li> + <li>Stefan Bunk</li> + <li>Paris Carbone</li> + <li>Ufuk Celebi</li> + <li>Nils Engelbach</li> + <li>Stephan Ewen</li> + <li>Gyula Fora</li> + <li>Gabor Hermann</li> + <li>Fabian Hueske</li> + <li>Vasiliki Kalavri</li> + <li>Johannes Kirschnick</li> + <li>Aljoscha Krettek</li> + <li>Suneel Marthi</li> + <li>Robert Metzger</li> + <li>Felix Neutatz</li> + <li>Chiwan Park</li> + <li>Flavio Pompermaier</li> + <li>Mingliang Qi</li> + <li>Shiva Teja Reddy</li> + <li>Till Rohrmann</li> + <li>Henry Saputra</li> + <li>Kousuke Saruta</li> + <li>Chesney Schepler</li> + <li>Erich Schubert</li> + <li>Peter Szabo</li> + <li>Jonas Traub</li> + <li>Kostas Tzoumas</li> + <li>Timo Walther</li> + <li>Daniel Warneke</li> + <li>Chen Xu</li> +</ul> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/9ec0a879/content/news/2015/02/04/january-in-flink.html ---------------------------------------------------------------------- diff --git a/content/news/2015/02/04/january-in-flink.html b/content/news/2015/02/04/january-in-flink.html new file mode 100644 index 0000000..2e7f6c6 --- /dev/null +++ b/content/news/2015/02/04/january-in-flink.html @@ -0,0 +1,241 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>Apache Flink: January 2015 in the Flink community</title> + <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> + <link rel="icon" href="/favicon.ico" type="image/x-icon"> + + <!-- Bootstrap --> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> + <link rel="stylesheet" href="/css/flink.css"> + <link rel="stylesheet" href="/css/syntax.css"> + + <!-- Blog RSS feed --> + <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <!-- We need to load Jquery in the header for custom google analytics event tracking--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> + <![endif]--> + </head> + <body> + + + <!-- Main content. --> + <div class="container"> + <div class="row"> + + + <div id="sidebar" class="col-sm-3"> + <!-- Top navbar. --> + <nav class="navbar navbar-default"> + <!-- The logo. --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-logo"> + <a href="/"> + <img alt="Apache Flink" src="/img/navbar-brand-logo.png" width="147px" height="73px"> + </a> + </div> + </div><!-- /.navbar-header --> + + <!-- The navigation links. --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-main"> + + <!-- Downloads --> + <li class=""><a class="btn btn-info" href="/downloads.html">Download Flink</a></li> + + <!-- Overview --> + <li><a href="/index.html">Home</a></li> + + <!-- Intro --> + <li><a href="/introduction.html">Introduction to Flink</a></li> + + <!-- Use cases --> + <li><a href="/usecases.html">Flink Use Cases</a></li> + + <!-- Powered by --> + <li><a href="/poweredby.html">Powered by Flink</a></li> + + <!-- Ecosystem --> + <li><a href="/ecosystem.html">Ecosystem</a></li> + + <!-- Community --> + <li><a href="/community.html">Community & Project Info</a></li> + + <!-- Contribute --> + <li><a href="/how-to-contribute.html">How to Contribute</a></li> + + <!-- Blog --> + <li class=" active hidden-md hidden-sm"><a href="/blog/"><b>Flink Blog</b></a></li> + + <hr /> + + + + <!-- Documentation --> + <!-- <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">Documentation <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> --> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation + <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1"; target="_blank">1.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2"; target="_blank">1.2 (Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + </ul> + </li> + + <!-- Quickstart --> + <li> + <a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html"; target="_blank">Quickstart <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + <!-- GitHub --> + <li> + <a href="https://github.com/apache/flink"; target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> + </li> + + + + + + + </ul> + + + + <ul class="nav navbar-nav navbar-bottom"> + <hr /> + + <!-- FAQ --> + <li ><a href="/faq.html">Project FAQ</a></li> + + <!-- Twitter --> + <li><a href="https://twitter.com/apacheflink"; target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + <!-- Visualizer --> + <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> + + </ul> + </div><!-- /.navbar-collapse --> + </nav> + + </div> + <div class="col-sm-9"> + <div class="row-fluid"> + <div class="col-sm-12"> + <div class="row"> + <h1>January 2015 in the Flink community</h1> + + <article> + <p>04 Feb 2015</p> + +<p>Happy 2015! Here is a (hopefully digestible) summary of what happened last month in the Flink community.</p> + +<h3 id="release">0.8.0 release</h3> + +<p>Flink 0.8.0 was released. See <a href="http://flink.apache.org/news/2015/01/21/release-0.8.html";>here</a> for the release notes.</p> + +<h3 id="flink-roadmap">Flink roadmap</h3> + +<p>The community has published a <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Roadmap";>roadmap for 2015</a> on the Flink wiki. Check it out to see what is coming up in Flink, and pick up an issue to contribute!</p> + +<h3 id="articles-in-the-press">Articles in the press</h3> + +<p>The Apache Software Foundation <a href="https://blogs.apache.org/foundation/entry/the_apache_software_foundation_announces69";>announced</a> Flink as a Top-Level Project. The announcement was picked up by the media, e.g., <a href="http://sdtimes.com/inside-apache-software-foundations-newest-top-level-project-apache-flink/?utm_content=11232092&utm_medium=social&utm_source=twitter";>here</a>, <a href="http://www.datanami.com/2015/01/12/apache-flink-takes-route-distributed-data-processing/";>here</a>, and <a href="http://i-programmer.info/news/197-data-mining/8176-flink-reaches-top-level-status.html";>here</a>.</p> + +<h3 id="hadoop-summit">Hadoop Summit</h3> + +<p>A submitted abstract on Flink Streaming won the community vote at âThe Future of Hadoopâ track.</p> + +<h3 id="meetups-and-talks">Meetups and talks</h3> + +<p>Flink was presented at the <a href="http://www.meetup.com/Hadoop-User-Group-France/events/219778022/";>Paris Hadoop User Group</a>, the <a href="http://www.meetup.com/hadoop/events/167785202/";>Bay Area Hadoop User Group</a>, the <a href="http://www.meetup.com/Apache-Tez-User-Group/events/219302692/";>Apache Tez User Group</a>, and <a href="https://fosdem.org/2015/schedule/track/graph_processing/";>FOSDEM 2015</a>. The January <a href="http://www.meetup.com/Apache-Flink-Meetup/events/219639984/";>Flink meetup in Berlin</a> had talks on recent community updates and new features.</p> + +<h2 id="notable-code-contributions">Notable code contributions</h2> + +<p><strong>Note:</strong> Code contributions listed here may not be part of a release or even the Flink master repository yet.</p> + +<h3 id="using-off-heap-memoryhttpsgithubcomapacheflinkpull290"><a href="https://github.com/apache/flink/pull/290";>Using off-heap memory</a></h3> + +<p>This pull request enables Flink to use off-heap memory for its internal memory uses (sort, hash, caching of intermediate data sets).</p> + +<h3 id="gelly-flinks-graph-apihttpsgithubcomapacheflinkpull335"><a href="https://github.com/apache/flink/pull/335";>Gelly, Flinkâs Graph API</a></h3> + +<p>This pull request introduces Gelly, Flinkâs brand new Graph API. Gelly offers a native graph programming abstraction with functionality for vertex-centric programming, as well as available graph algorithms. See <a href="http://www.slideshare.net/vkalavri/largescale-graph-processing-with-apache-flink-graphdevroom-fosdem15";>this slide set</a> for an overview of Gelly.</p> + +<h3 id="semantic-annotationshttpsgithubcomapacheflinkpull311"><a href="https://github.com/apache/flink/pull/311";>Semantic annotations</a></h3> + +<p>Semantic annotations are a powerful mechanism to expose information about the behavior of Flink functions to Flinkâs optimizer. The optimizer can leverage this information to generate more efficient execution plans. For example the output of a Reduce operator that groups on the second field of a tuple is still partitioned on that field if the Reduce function does not modify the value of the second field. By exposing this information to the optimizer, the optimizer can generate plans that avoid expensive data shuffling and reuse the partitioned output of Reduce. Semantic annotations can be defined for most data types, including (nested) tuples and POJOs. See the snapshot documentation for details (not online yet).</p> + +<h3 id="new-yarn-clienthttpsgithubcomapacheflinkpull292"><a href="https://github.com/apache/flink/pull/292";>New YARN client</a></h3> + +<p>The improved YARN client of Flink now allows users to deploy Flink on YARN for executing a single job. Older versions only supported a long-running YARN session. The code of the YARN client has been refactored to provide an (internal) Java API for controlling YARN clusters more easily.</p> + + </article> + </div> + + <div class="row"> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + </div> + </div> +</div> + </div> + </div> + + <hr /> + + <div class="row"> + <div class="footer text-center col-sm-12"> + <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> + <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> + <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> + </div> + </div> + </div><!-- /.container --> + + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> + <script src="/js/codetabs.js"></script> + <script src="/js/stickysidebar.js"></script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-52545728-1', 'auto'); + ga('send', 'pageview'); + </script> + </body> +</html>
