This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/asf-site by this push:
new deddf69 Publishing website 2020/12/14 12:03:04 at commit fc42781
deddf69 is described below
commit deddf690f4340c5c711c3084b1e3b2bf332f2c49
Author: jenkins <bui...@apache.org>
AuthorDate: Mon Dec 14 12:03:04 2020 +0000
Publishing website 2020/12/14 12:03:04 at commit fc42781
---
.../2020/12/14/splittable-do-fn-is-available.html | 1 +
website/generated-content/blog/index.html | 4 +-
website/generated-content/blog/index.xml | 343 +++++----------------
.../blog/splittable-do-fn-is-available/index.html | 43 +++
.../generated-content/categories/blog/index.xml | 343 +++++----------------
website/generated-content/categories/index.xml | 2 +-
website/generated-content/feed.xml | 184 +++++------
website/generated-content/index.html | 2 +-
website/generated-content/sitemap.xml | 2 +-
9 files changed, 302 insertions(+), 622 deletions(-)
diff --git
a/website/generated-content/blog/2020/12/14/splittable-do-fn-is-available.html
b/website/generated-content/blog/2020/12/14/splittable-do-fn-is-available.html
new file mode 100644
index 0000000..ec4d5b5
--- /dev/null
+++
b/website/generated-content/blog/2020/12/14/splittable-do-fn-is-available.html
@@ -0,0 +1 @@
+<!doctype
html><html><head><title>/blog/splittable-do-fn-is-available/</title><link
rel=canonical href=/blog/splittable-do-fn-is-available/><meta name=robots
content="noindex"><meta charset=utf-8><meta http-equiv=refresh content="0;
url=/blog/splittable-do-fn-is-available/"></head></html>
\ No newline at end of file
diff --git a/website/generated-content/blog/index.html
b/website/generated-content/blog/index.html
index c154da7..34fc015 100644
--- a/website/generated-content/blog/index.html
+++ b/website/generated-content/blog/index.html
@@ -1,7 +1,9 @@
<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta
http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport
content="width=device-width,initial-scale=1"><title>Blogs</title><meta
name=description content="Apache Beam is an open source, unified model and set
of language-specific SDKs for defining and executing data processing workflows,
and also data ingestion and integration flows, supporting Enterprise
Integration Patterns (EIPs) and Domain Specific Languag [...]
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span><span class=icon-bar></span><span
class=icon-bar></span></button>
-<a href=/ class=navbar-brand><img alt=Brand style=height:25px
src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask
closed"></div><div id=navbar class="navbar-container closed"><ul class="nav
navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a
href=/documentation/>Documentation</a></li><li><a
href=/documentation/sdks/java/>Languages</a></li><li><a
href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a
href=/roadmap/>Roadmap</a></li>< [...]
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px
src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask
closed"></div><div id=navbar class="navbar-container closed"><ul class="nav
navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a
href=/documentation/>Documentation</a></li><li><a
href=/documentation/sdks/java/>Languages</a></li><li><a
href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a
href=/roadmap/>Roadmap</a></li>< [...]
+•
+Boyuan Zhang</i></p>We are pleased to announce that Splittable DoFn (SDF) is
ready for use in the Beam Python, Java, and Go SDKs for versions 2.<p><a
class="btn btn-default btn-sm" href=/blog/splittable-do-fn-is-available/
role=button>Read more <span class="glyphicon glyphicon-menu-right"
aria-hidden=true></span></a></p><hr><h3><a class=post-link
href=/blog/beam-2.26.0/>Apache Beam 2.26.0</a></h3><p><i>Dec 11, 2020
•
Robert Burke [<a href=https://twitter.com/lostluck>@lostluck</a>]</i></p>We
are happy to present the new 2.<p><a class="btn btn-default btn-sm"
href=/blog/beam-2.26.0/ role=button>Read more <span class="glyphicon
glyphicon-menu-right" aria-hidden=true></span></a></p><hr><h3><a
class=post-link href=/blog/beam-2.25.0/>Apache Beam 2.25.0</a></h3><p><i>Oct
23, 2020
•
diff --git a/website/generated-content/blog/index.xml
b/website/generated-content/blog/index.xml
index 8023ba4..ffbec54 100644
--- a/website/generated-content/blog/index.xml
+++ b/website/generated-content/blog/index.xml
@@ -1,4 +1,86 @@
-<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
Blogs</title><link>/blog/</link><description>Recent content in Blogs on Apache
Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri,
11 Dec 2020 12:00:00 -0800</lastBuildDate><atom:link href="/blog/index.xml"
rel="self" type="application/rss+xml"/><item><title>Blog: Apache Beam
2.26.0</title><link>/blog/beam-2.26.0/</link><pubDate>Fri, 11 Dec 2020 12:00:00
-0800</pubDate><gui [...]
+<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
Blogs</title><link>/blog/</link><description>Recent content in Blogs on Apache
Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Mon,
14 Dec 2020 00:00:01 -0800</lastBuildDate><atom:link href="/blog/index.xml"
rel="self" type="application/rss+xml"/><item><title>Blog: Splittable DoFn in
Apache Beam is Ready to
Use</title><link>/blog/splittable-do-fn-is-available/</link><pubDate [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>We are pleased to announce that Splittable DoFn (SDF) is ready for use
in the Beam Python, Java,
+and Go SDKs for versions 2.25.0 and later.</p>
+<p>In 2017, <a
href="https://beam.apache.org/blog/splittable-do-fn/";>Splittable DoFn Blog
Post</a> proposed
+to build <a href="https://s.apache.org/splittable-do-fn";>Splittable
DoFn</a> APIs as the new recommended way of
+building I/O connectors. Splittable DoFn is a generalization of
<code>DoFn</code> that gives it the core
+capabilities of <code>Source</code> while retaining
<code>DoFn</code>'s syntax, flexibility, modularity, and ease of
+coding. Thus, it becomes much easier to develop complex I/O connectors with
simpler and reusable
+code.</p>
+<p>SDF has three advantages over the existing
<code>UnboundedSource</code> and <code>BoundedSource</code>:</p>
+<ul>
+<li>SDF provides a unified set of APIs to handle both unbounded and bounded
cases.</li>
+<li>SDF enables reading from source descriptors dynamically.
+<ul>
+<li>Taking KafkaIO as an example, within
<code>UnboundedSource</code>/<code>BoundedSource</code> API, you
must specify
+the topic and partition you want to read from during pipeline construction
time. There is no way
+for <code>UnboundedSource</code>/<code>BoundedSource</code> to
accept topics and partitions as inputs during execution
+time. But it&rsquo;s built-in to SDF.</li>
+</ul>
+</li>
+<li>SDF fits in as any node on a pipeline freely with the ability of
splitting.
+<ul>
+<li><code>UnboundedSource</code>/<code>BoundedSource</code> has
to be the root node of the pipeline to gain performance
+benefits from splitting strategies, which limits many real-world usages. This
is no longer a limit
+for an SDF.</li>
+</ul>
+</li>
+</ul>
+<p>As SDF is now ready to use with all the mentioned improvements, it is
the recommended
+way to build the new I/O connectors. Try out building your own Splittable DoFn
by following the
+<a
href="https://beam.apache.org/documentation/programming-guide/#splittable-dofns";>programming
guide</a>. We
+have provided tonnes of common utility classes such as common types of
<code>RestrictionTracker</code> and
+<code>WatermarkEstimator</code> in Beam SDK, which will help you onboard
easily. As for the existing I/O
+connectors, we have wrapped <code>UnboundedSource</code> and
<code>BoundedSource</code> implementations into Splittable
+DoFns, yet we still encourage developers to convert
<code>UnboundedSource</code>/<code>BoundedSource</code> into actual
+Splittable DoFn implementation to gain more performance benefits.</p>
+<p>Many thanks to every contributor who brought this highly anticipated
design into the data processing
+world. We are really excited to see that users benefit from SDF.</p>
+<p>Below are some real-world SDF examples for you to explore.</p>
+<h2 id="real-world-splittable-dofn-examples">Real world Splittable DoFn
examples</h2>
+<p><strong>Java Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java#L118";>Kafka</a>:
+An I/O connector for <a href="https://kafka.apache.org/";>Apache Kafka</a>
+(an open-source distributed event streaming platform).</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java#L787";>Watch</a>:
+Uses a polling function producing a growing set of outputs for each input
until a per-input
+termination condition is met.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java#L365";>Parquet</a>:
+An I/O connector for <a href="https://parquet.apache.org/";>Apache
Parquet</a>
+(an open-source columnar storage format).</li>
+<li><a
href="https://github.com/apache/beam/blob/6fdde4f4eab72b49b10a8bb1cb3be263c5c416b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java#L493";>HL7v2</a>:
+An I/O connector for HL7v2 messages (a clinical messaging format that provides
data about events
+that occur inside an organization) part of
+<a href="https://cloud.google.com/healthcare";>Google’s Cloud Healthcare
API</a>.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L248";>BoundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L432";>UnboundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html";>UnboundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Python Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/python/apache_beam/io/iobase.py#L1375";>BoundedSourceWrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.iobase.html#apache_beam.io.iobase.BoundedSource";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Go Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/ce190e11332469ea59b6c9acf16ee7c673ccefdd/sdks/go/pkg/beam/io/textio/sdf.go#L40";>textio.ReadSdf</a>
implements reading from text files using a splittable DoFn.</li>
+</ul></description></item><item><title>Blog: Apache Beam
2.26.0</title><link>/blog/beam-2.26.0/</link><pubDate>Fri, 11 Dec 2020 12:00:00
-0800</pubDate><guid>/blog/beam-2.26.0/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -5190,261 +5272,4 @@ limitations under the License.
<p>Embedded mode is useful for development and debugging. Apex in
production runs distributed on Apache Hadoop YARN clusters. An example how a
Beam pipeline can be embedded into an Apex application package to run on YARN
can be found <a
href="https://github.com/tweise/apex-samples/tree/master/beam-apex-wordcount";>here</a>
and support for direct launch in the runner is currently being worked
on.</p>
<p>The Beam project has a strong focus on development process and tooling,
including testing. For the runners, there is a comprehensive test suite with
more than 200 integration tests that are executed against each runner to ensure
they don’t break as changes are made. The tests cover the capabilities of the
matrix and thus are a measure of completeness and correctness of the runner
implementations. The suite was very helpful when developing the Apex
runner.</p>
<h2 id="outlook">Outlook</h2>
-<p>The next step is to take the Apex runner from functional to ready for
real applications that run distributed, leveraging the scalability and
performance features of Apex, similar to its native API. This includes chaining
of ParDos, partitioning, optimizing combine operations etc. To get involved,
please see <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20and%20component%20%3D%20runner-apex%20and%20resolution%20%3D%20unresolved";>JIRA</a>
and join the [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>The Beam Programming Model unifies writing pipelines for Batch and
Streaming
-pipelines. We’ve recently introduced a new PTransform to write tests for
-pipelines that will be run over unbounded datasets and must handle out-of-order
-and delayed data.</p>
-<p>Watermarks, Windows and Triggers form a core part of the Beam
programming model
-&ndash; they respectively determine how your data are grouped, when your
input is
-complete, and when to produce results. This is true for all pipelines,
-regardless of if they are processing bounded or unbounded inputs. If you’re not
-familiar with watermarks, windowing, and triggering in the Beam model,
-<a
href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101";>Streaming
101</a>
-and <a
href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-102";>Streaming
102</a>
-are an excellent place to get started. A key takeaway from
-these articles: in realistic streaming scenarios with intermittent failures and
-disconnected users, data can arrive out of order or be delayed. Beam’s
-primitives provide a way for users to perform useful, powerful, and correct
-computations in spite of these challenges.</p>
-<p>As Beam pipeline authors, we need comprehensive tests that cover crucial
-failure scenarios and corner cases to gain real confidence that a pipeline is
-ready for production. The existing testing infrastructure within the Beam SDKs
-permits tests to be written which examine the contents of a Pipeline at
-execution time. However, writing unit tests for pipelines that may receive
-late data or trigger multiple times has historically ranged from complex to
-not possible, as pipelines that read from unbounded sources do not shut down
-without external intervention, while pipelines that read from bounded sources
-exclusively cannot test behavior with late data nor most speculative triggers.
-Without additional tools, pipelines that use custom triggers and handle
-out-of-order data could not be easily tested.</p>
-<p>This blog post introduces our new framework for writing tests for
pipelines that
-handle delayed and out-of-order data in the context of the LeaderBoard pipeline
-from the Mobile Gaming example series.</p>
-<h2 id="leaderboard-and-the-mobile-gaming-example">LeaderBoard and the
Mobile Gaming Example</h2>
-<p><a
href="https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java#L177";>LeaderBoard</a>
-is part of the <a
href="https://github.com/apache/beam/tree/master/examples/java/src/main/java/org/apache/beam/examples/complete/game";>Beam
mobile gaming examples</a>
-(and <a href="/get-started/mobile-gaming-example/">walkthroughs</a>)
-which produces a continuous accounting of user and team scores. User scores are
-calculated over the lifetime of the program, while team scores are calculated
-within fixed windows with a default duration of one hour. The LeaderBoard
-pipeline produces speculative and late panes as appropriate, based on the
-configured triggering and allowed lateness of the pipeline. The expected
outputs
-of the LeaderBoard pipeline vary depending on when elements arrive in relation
-to the watermark and the progress of processing time, which could not
previously
-be controlled within a test.</p>
-<h2 id="writing-deterministic-tests-to-emulate-nondeterminism">Writing
Deterministic Tests to Emulate Nondeterminism</h2>
-<p>The Beam testing infrastructure provides the
-<a
href="https://beam.apache.org/releases/javadoc/2.26.0/org/apache/beam/sdk/testing/PAssert.html";>PAssert</a>
-methods, which assert properties about the contents of a PCollection from
within
-a pipeline. We have expanded this infrastructure to include
-<a
href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java";>TestStream</a>,
-which is a PTransform that performs a series of events, consisting of adding
-additional elements to a pipeline, advancing the watermark of the TestStream,
-and advancing the pipeline processing time clock. TestStream permits tests
which
-observe the effects of triggers on the output a pipeline produces.</p>
-<p>While executing a pipeline that reads from a TestStream, the read waits
for all
-of the consequences of each event to complete before continuing on to the next
-event, ensuring that when processing time advances, triggers that are based on
-processing time fire as appropriate. With this transform, the effect of
-triggering and allowed lateness can be observed on a pipeline, including
-reactions to speculative and late panes and dropped data.</p>
-<h2 id="element-timings">Element Timings</h2>
-<p>Elements arrive either before, with, or after the watermark, which
categorizes
-them into the &ldquo;early&rdquo;, &ldquo;on-time&rdquo;, and
&ldquo;late&rdquo; divisions. &ldquo;Late&rdquo; elements can be
-further subdivided into &ldquo;unobservably&rdquo;,
&ldquo;observably&rdquo;, and &ldquo;droppably&rdquo; late,
-depending on the window to which they are assigned and the maximum allowed
-lateness, as specified by the windowing strategy. Elements that arrive with
-these timings are emitted into panes, which can be
&ldquo;EARLY&rdquo;, &ldquo;ON-TIME&rdquo;, or
-&ldquo;LATE&rdquo;, depending on the position of the watermark when
the pane was emitted.</p>
-<p>Using TestStream, we can write tests that demonstrate that speculative
panes are
-output after their trigger condition is met, that the advancing of the
watermark
-causes the on-time pane to be produced, and that late-arriving data produces
-refinements when it arrives before the maximum allowed lateness, and is dropped
-after.</p>
-<p>The following examples demonstrate how you can use TestStream to provide
a
-sequence of events to the Pipeline, where the arrival of elements is
interspersed
-with updates to the watermark and the advance of processing time. Each of these
-events runs to completion before additional events occur.</p>
-<p>In the diagrams, the time at which events occurred in
&ldquo;real&rdquo; (event) time
-progresses as the graph moves to the right. The time at which the pipeline
-receives them progresses as the graph goes upwards. The watermark is
represented
-by the squiggly red line, and each starburst is the firing of a trigger and the
-associated pane.</p>
-<img class="center-block"
src="/images/blog/test-stream/elements-all-on-time.png" alt="Elements on the
Event and Processing time axes, with the Watermark and produced panes"
width="442">
-<h3 id="everything-arrives-on-time">Everything arrives on-time</h3>
-<p>For example, if we create a TestStream where all the data arrives before
the
-watermark and provide the result PCollection as input to the
CalculateTeamScores
-PTransform:</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant& [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class= [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class= [...]
- <span class="c1">// Move the watermark past the end the end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<p>we can then assert that the result PCollection contains elements that
arrived:</p>
-<img class="center-block"
src="/images/blog/test-stream/elements-all-on-time.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// Only one value is
emitted for the blue team
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="some-elements-are-late-but-arrive-before-the-end-of-the-window">Some
elements are late, but arrive before the end of the window</h3>
-<p>We can also add data to the TestStream after the watermark, but before
the end
-of the window (shown below to the left of the red watermark), which
demonstrates
-&ldquo;unobservably late&rdquo; data - that is, data that arrives
late, but is promoted by
-the system to be on time, as it arrives before the watermark passes the end of
-the window</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant&l [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).&l [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-unobservably-late.png" alt="An element
arrives late, but before the watermark passes the end of the window, and is
produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// Only one value is
emitted for the blue team
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3 id="elements-are-late-and-arrive-after-the-end-of-the-window">Elements
are late, and arrive after the end of the window</h3>
-<p>By advancing the watermark farther in time before adding the late data,
we can
-demonstrate the triggering behavior that causes the system to emit an on-time
-pane, and then after the late data arrives, a pane that refines the
result.</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant&l [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">). [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-observably-late.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// An on-time pane
is emitted with the events that arrived before the window closed
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inOnTimePane</span><span class="o">(</span><span
class="n">window</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">6</span><span class="o">));</span>
-<span class="c1">// The final pane contains the late refinement
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inFinalPane</span><span class="o">(</span><span
class="n">window</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="elements-are-late-and-after-the-end-of-the-window-plus-the-allowed-lateness">Elements
are late, and after the end of the window plus the allowed lateness</h3>
-<p>If we push the watermark even further into the future, beyond the maximum
-configured allowed lateness, we can demonstrate that the late element is
dropped
-by the system.</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span c [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">ALLOWED_LATENESS</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span>
- <span
class="s">&#34;sky&#34;</span><span class="o">,</span>
- <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
- <span class="n">12</span><span
class="o">,</span>
- <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">).</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-droppably-late.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// An on-time pane
is emitted with the events that arrived before the window closed
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">6</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="elements-arrive-before-the-end-of-the-window-and-some-processing-time-passes">Elements
arrive before the end of the window, and some processing time passes</h3>
-<p>Using additional methods, we can demonstrate the behavior of speculative
-triggers by advancing the processing time of the TestStream. If we add elements
-to an input PCollection, occasionally advancing the processing time clock, and
-apply <code>CalculateUserScores</code></p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="n">GameActionInfo</span><span class="o">.</span><span
class="na">class</span><span class="o">))& [...]
- <span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span class="n">Ins
[...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">2</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class [...]
-<span class="o">.</span><span
class="na">advanceProcessingTime</span><span
class="o">(</span><span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span class="n">12</span><span
class="o">))</span>
- <span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span
class="s">&#34;oxblood&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">2</span><span
class="o">,</span> <span class="k">new</span> <span class="n">Ins
[...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">4</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class [...]
-<span class="o">.</span><span
class="na">advanceProcessingTime</span><span
class="o">(</span><span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span class="n">15</span><span
class="o">))</span>
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">userScores</span>
<span class="o">=</span>
- <span class="n">p</span><span class="o">.</span><span
class="na">apply</span><span class="o">(</span><span
class="n">infos</span><span class="o">).</span><span
class="na">apply</span><span class="o">(</span><span
class="k">new</span> <span
class="n">CalculateUserScores</span><span class="o">(</span><span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-processing-speculative.png"
alt="Elements all arrive before the watermark, and are produced in the on-time
pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">userScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inEarlyGlobalWindowPanes</span><span class="o">()</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="n">5</span><span class="o">),</span>
- <span class="n">KV</span><span
class="o">.</span><span class="na">of</span><span
class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="n">9</span><span class="o">),</span>
-<span class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;oxblood&#34;</span><span class="o">,</span>
<span class="n">2</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h2 id="teststream---under-the-hood">TestStream - Under the Hood</h2>
-<p>TestStream relies on a pipeline concept we’ve introduced, called
quiescence, to
-utilize the existing runner infrastructure while providing guarantees about
when
-a root transform will called by the runner. This consists of properties about
-pending elements and triggers, namely:</p>
-<ul>
-<li>No trigger is permitted to fire but has not fired</li>
-<li>All elements are either buffered in state or cannot progress until a
side input becomes available</li>
-</ul>
-<p>Simplified, this means that, in the absence of an advancement in input
-watermarks or processing time, or additional elements being added to the
-pipeline, the pipeline will not make progress. Whenever the TestStream
PTransform
-performs an action, the runner must not reinvoke the same instance until the
-pipeline has quiesced. This ensures that the events specified by TestStream
-happen &ldquo;in-order&rdquo;, which ensures that input watermarks and
the system clock do
-not advance ahead of the elements they hoped to hold up.</p>
-<p>The DirectRunner has been modified to use quiescence as the signal that
it
-should add more work to the Pipeline, and the implementation of TestStream in
-that runner uses this fact to perform a single output per event. The
DirectRunner
-implementation also directly controls the runner’s system clock, ensuring that
-tests will complete promptly even if there is a multi-minute processing time
-trigger located within the pipeline.</p>
-<p>The TestStream transform is supported in the DirectRunner. For most
users, tests
-written using TestPipeline and PAsserts will automatically function while using
-TestStream.</p>
-<h2 id="summary">Summary</h2>
-<p>The addition of TestStream alongside window and pane-specific matchers
in PAssert
-has enabled the testing of Pipelines which produce speculative and late panes.
-This permits tests for all styles of pipeline to be expressed directly within
the
-Java SDK. If you have questions or comments, we’d love to hear them on the
-<a href="/get-started/support/">mailing
lists</a>.</p></description></item></channel></rss>
\ No newline at end of file
+<p>The next step is to take the Apex runner from functional to ready for
real applications that run distributed, leveraging the scalability and
performance features of Apex, similar to its native API. This includes chaining
of ParDos, partitioning, optimizing combine operations etc. To get involved,
please see <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20and%20component%20%3D%20runner-apex%20and%20resolution%20%3D%20unresolved";>JIRA</a>
and join the [...]
\ No newline at end of file
diff --git
a/website/generated-content/blog/splittable-do-fn-is-available/index.html
b/website/generated-content/blog/splittable-do-fn-is-available/index.html
new file mode 100644
index 0000000..d53156d
--- /dev/null
+++ b/website/generated-content/blog/splittable-do-fn-is-available/index.html
@@ -0,0 +1,43 @@
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta
http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport
content="width=device-width,initial-scale=1"><title>Splittable DoFn in Apache
Beam is Ready to Use</title><meta name=description content="Apache Beam is an
open source, unified model and set of language-specific SDKs for defining and
executing data processing workflows, and also data ingestion and integration
flows, supporting Enterprise Integration Pa [...]
+<span class=sr-only>Toggle navigation</span>
+<span class=icon-bar></span><span class=icon-bar></span><span
class=icon-bar></span></button>
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px
src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask
closed"></div><div id=navbar class="navbar-container closed"><ul class="nav
navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a
href=/documentation/>Documentation</a></li><li><a
href=/documentation/sdks/java/>Languages</a></li><li><a
href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a
href=/roadmap/>Roadmap</a></li>< [...]
+•
+Boyuan Zhang</p></header><div class=post-content itemprop=articleBody><p>We
are pleased to announce that Splittable DoFn (SDF) is ready for use in the Beam
Python, Java,
+and Go SDKs for versions 2.25.0 and later.</p><p>In 2017, <a
href=https://beam.apache.org/blog/splittable-do-fn/>Splittable DoFn Blog
Post</a> proposed
+to build <a href=https://s.apache.org/splittable-do-fn>Splittable DoFn</a>
APIs as the new recommended way of
+building I/O connectors. Splittable DoFn is a generalization of
<code>DoFn</code> that gives it the core
+capabilities of <code>Source</code> while retaining <code>DoFn</code>'s
syntax, flexibility, modularity, and ease of
+coding. Thus, it becomes much easier to develop complex I/O connectors with
simpler and reusable
+code.</p><p>SDF has three advantages over the existing
<code>UnboundedSource</code> and <code>BoundedSource</code>:</p><ul><li>SDF
provides a unified set of APIs to handle both unbounded and bounded
cases.</li><li>SDF enables reading from source descriptors
dynamically.<ul><li>Taking KafkaIO as an example, within
<code>UnboundedSource</code>/<code>BoundedSource</code> API, you must specify
+the topic and partition you want to read from during pipeline construction
time. There is no way
+for <code>UnboundedSource</code>/<code>BoundedSource</code> to accept topics
and partitions as inputs during execution
+time. But it’s built-in to SDF.</li></ul></li><li>SDF fits in as any
node on a pipeline freely with the ability of
splitting.<ul><li><code>UnboundedSource</code>/<code>BoundedSource</code> has
to be the root node of the pipeline to gain performance
+benefits from splitting strategies, which limits many real-world usages. This
is no longer a limit
+for an SDF.</li></ul></li></ul><p>As SDF is now ready to use with all the
mentioned improvements, it is the recommended
+way to build the new I/O connectors. Try out building your own Splittable DoFn
by following the
+<a
href=https://beam.apache.org/documentation/programming-guide/#splittable-dofns>programming
guide</a>. We
+have provided tonnes of common utility classes such as common types of
<code>RestrictionTracker</code> and
+<code>WatermarkEstimator</code> in Beam SDK, which will help you onboard
easily. As for the existing I/O
+connectors, we have wrapped <code>UnboundedSource</code> and
<code>BoundedSource</code> implementations into Splittable
+DoFns, yet we still encourage developers to convert
<code>UnboundedSource</code>/<code>BoundedSource</code> into actual
+Splittable DoFn implementation to gain more performance benefits.</p><p>Many
thanks to every contributor who brought this highly anticipated design into the
data processing
+world. We are really excited to see that users benefit from SDF.</p><p>Below
are some real-world SDF examples for you to explore.</p><h2
id=real-world-splittable-dofn-examples>Real world Splittable DoFn
examples</h2><p><strong>Java Examples</strong></p><ul><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java#L118>Kafka</a>:
+An I/O connector for <a href=https://kafka.apache.org/>Apache Kafka</a>
+(an open-source distributed event streaming platform).</li><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java#L787>Watch</a>:
+Uses a polling function producing a growing set of outputs for each input
until a per-input
+termination condition is met.</li><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java#L365>Parquet</a>:
+An I/O connector for <a href=https://parquet.apache.org/>Apache Parquet</a>
+(an open-source columnar storage format).</li><li><a
href=https://github.com/apache/beam/blob/6fdde4f4eab72b49b10a8bb1cb3be263c5c416b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java#L493>HL7v2</a>:
+An I/O connector for HL7v2 messages (a clinical messaging format that provides
data about events
+that occur inside an organization) part of
+<a href=https://cloud.google.com/healthcare>Google’s Cloud Healthcare
API</a>.</li><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L248>BoundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href=https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html>BoundedSource</a>
+implementation to a splittable DoFn.</li><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L432>UnboundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href=https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html>UnboundedSource</a>
+implementation to a splittable DoFn.</li></ul><p><strong>Python
Examples</strong></p><ul><li><a
href=https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/python/apache_beam/io/iobase.py#L1375>BoundedSourceWrapper</a>:
+A wrapper which converts an existing <a
href=https://beam.apache.org/releases/pydoc/current/apache_beam.io.iobase.html#apache_beam.io.iobase.BoundedSource>BoundedSource</a>
+implementation to a splittable DoFn.</li></ul><p><strong>Go
Examples</strong></p><ul><li><a
href=https://github.com/apache/beam/blob/ce190e11332469ea59b6c9acf16ee7c673ccefdd/sdks/go/pkg/beam/io/textio/sdf.go#L40>textio.ReadSdf</a>
implements reading from text files using a splittable
DoFn.</li></ul></div></article></div><footer class=footer><div
class=footer__contained><div class=footer__cols><div
class=footer__cols__col><div class=footer__cols__col__logo><img
src=/images/beam_logo_circl [...]
+<a href=http://www.apache.org>The Apache Software Foundation</a>
+| <a href=/privacy_policy>Privacy Policy</a>
+| <a href=/feed.xml>RSS Feed</a><br><br>Apache Beam, Apache, Beam, the Beam
logo, and the Apache feather logo are either registered trademarks or
trademarks of The Apache Software Foundation. All other products or name brands
are trademarks of their respective holders, including The Apache Software
Foundation.</div></footer></body></html>
\ No newline at end of file
diff --git a/website/generated-content/categories/blog/index.xml
b/website/generated-content/categories/blog/index.xml
index 88c476e..ce71d5b 100644
--- a/website/generated-content/categories/blog/index.xml
+++ b/website/generated-content/categories/blog/index.xml
@@ -1,4 +1,86 @@
-<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
blog</title><link>/categories/blog/</link><description>Recent content in blog
on Apache Beam</description><generator>Hugo --
gohugo.io</generator><lastBuildDate>Fri, 11 Dec 2020 12:00:00
-0800</lastBuildDate><atom:link href="/categories/blog/index.xml" rel="self"
type="application/rss+xml"/><item><title>Blog: Apache Beam
2.26.0</title><link>/blog/beam-2.26.0/</link><pubDate>Fri, 11 Dec 2020 12:00:00
[...]
+<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
blog</title><link>/categories/blog/</link><description>Recent content in blog
on Apache Beam</description><generator>Hugo --
gohugo.io</generator><lastBuildDate>Mon, 14 Dec 2020 00:00:01
-0800</lastBuildDate><atom:link href="/categories/blog/index.xml" rel="self"
type="application/rss+xml"/><item><title>Blog: Splittable DoFn in Apache Beam
is Ready to Use</title><link>/blog/splittable-do-fn-is-avail [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>We are pleased to announce that Splittable DoFn (SDF) is ready for use
in the Beam Python, Java,
+and Go SDKs for versions 2.25.0 and later.</p>
+<p>In 2017, <a
href="https://beam.apache.org/blog/splittable-do-fn/";>Splittable DoFn Blog
Post</a> proposed
+to build <a href="https://s.apache.org/splittable-do-fn";>Splittable
DoFn</a> APIs as the new recommended way of
+building I/O connectors. Splittable DoFn is a generalization of
<code>DoFn</code> that gives it the core
+capabilities of <code>Source</code> while retaining
<code>DoFn</code>'s syntax, flexibility, modularity, and ease of
+coding. Thus, it becomes much easier to develop complex I/O connectors with
simpler and reusable
+code.</p>
+<p>SDF has three advantages over the existing
<code>UnboundedSource</code> and <code>BoundedSource</code>:</p>
+<ul>
+<li>SDF provides a unified set of APIs to handle both unbounded and bounded
cases.</li>
+<li>SDF enables reading from source descriptors dynamically.
+<ul>
+<li>Taking KafkaIO as an example, within
<code>UnboundedSource</code>/<code>BoundedSource</code> API, you
must specify
+the topic and partition you want to read from during pipeline construction
time. There is no way
+for <code>UnboundedSource</code>/<code>BoundedSource</code> to
accept topics and partitions as inputs during execution
+time. But it&rsquo;s built-in to SDF.</li>
+</ul>
+</li>
+<li>SDF fits in as any node on a pipeline freely with the ability of
splitting.
+<ul>
+<li><code>UnboundedSource</code>/<code>BoundedSource</code> has
to be the root node of the pipeline to gain performance
+benefits from splitting strategies, which limits many real-world usages. This
is no longer a limit
+for an SDF.</li>
+</ul>
+</li>
+</ul>
+<p>As SDF is now ready to use with all the mentioned improvements, it is
the recommended
+way to build the new I/O connectors. Try out building your own Splittable DoFn
by following the
+<a
href="https://beam.apache.org/documentation/programming-guide/#splittable-dofns";>programming
guide</a>. We
+have provided tonnes of common utility classes such as common types of
<code>RestrictionTracker</code> and
+<code>WatermarkEstimator</code> in Beam SDK, which will help you onboard
easily. As for the existing I/O
+connectors, we have wrapped <code>UnboundedSource</code> and
<code>BoundedSource</code> implementations into Splittable
+DoFns, yet we still encourage developers to convert
<code>UnboundedSource</code>/<code>BoundedSource</code> into actual
+Splittable DoFn implementation to gain more performance benefits.</p>
+<p>Many thanks to every contributor who brought this highly anticipated
design into the data processing
+world. We are really excited to see that users benefit from SDF.</p>
+<p>Below are some real-world SDF examples for you to explore.</p>
+<h2 id="real-world-splittable-dofn-examples">Real world Splittable DoFn
examples</h2>
+<p><strong>Java Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java#L118";>Kafka</a>:
+An I/O connector for <a href="https://kafka.apache.org/";>Apache Kafka</a>
+(an open-source distributed event streaming platform).</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java#L787";>Watch</a>:
+Uses a polling function producing a growing set of outputs for each input
until a per-input
+termination condition is met.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java#L365";>Parquet</a>:
+An I/O connector for <a href="https://parquet.apache.org/";>Apache
Parquet</a>
+(an open-source columnar storage format).</li>
+<li><a
href="https://github.com/apache/beam/blob/6fdde4f4eab72b49b10a8bb1cb3be263c5c416b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java#L493";>HL7v2</a>:
+An I/O connector for HL7v2 messages (a clinical messaging format that provides
data about events
+that occur inside an organization) part of
+<a href="https://cloud.google.com/healthcare";>Google’s Cloud Healthcare
API</a>.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L248";>BoundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L432";>UnboundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html";>UnboundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Python Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/python/apache_beam/io/iobase.py#L1375";>BoundedSourceWrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.iobase.html#apache_beam.io.iobase.BoundedSource";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Go Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/ce190e11332469ea59b6c9acf16ee7c673ccefdd/sdks/go/pkg/beam/io/textio/sdf.go#L40";>textio.ReadSdf</a>
implements reading from text files using a splittable DoFn.</li>
+</ul></description></item><item><title>Blog: Apache Beam
2.26.0</title><link>/blog/beam-2.26.0/</link><pubDate>Fri, 11 Dec 2020 12:00:00
-0800</pubDate><guid>/blog/beam-2.26.0/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -5190,261 +5272,4 @@ limitations under the License.
<p>Embedded mode is useful for development and debugging. Apex in
production runs distributed on Apache Hadoop YARN clusters. An example how a
Beam pipeline can be embedded into an Apex application package to run on YARN
can be found <a
href="https://github.com/tweise/apex-samples/tree/master/beam-apex-wordcount";>here</a>
and support for direct launch in the runner is currently being worked
on.</p>
<p>The Beam project has a strong focus on development process and tooling,
including testing. For the runners, there is a comprehensive test suite with
more than 200 integration tests that are executed against each runner to ensure
they don’t break as changes are made. The tests cover the capabilities of the
matrix and thus are a measure of completeness and correctness of the runner
implementations. The suite was very helpful when developing the Apex
runner.</p>
<h2 id="outlook">Outlook</h2>
-<p>The next step is to take the Apex runner from functional to ready for
real applications that run distributed, leveraging the scalability and
performance features of Apex, similar to its native API. This includes chaining
of ParDos, partitioning, optimizing combine operations etc. To get involved,
please see <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20and%20component%20%3D%20runner-apex%20and%20resolution%20%3D%20unresolved";>JIRA</a>
and join the [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>The Beam Programming Model unifies writing pipelines for Batch and
Streaming
-pipelines. We’ve recently introduced a new PTransform to write tests for
-pipelines that will be run over unbounded datasets and must handle out-of-order
-and delayed data.</p>
-<p>Watermarks, Windows and Triggers form a core part of the Beam
programming model
-&ndash; they respectively determine how your data are grouped, when your
input is
-complete, and when to produce results. This is true for all pipelines,
-regardless of if they are processing bounded or unbounded inputs. If you’re not
-familiar with watermarks, windowing, and triggering in the Beam model,
-<a
href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101";>Streaming
101</a>
-and <a
href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-102";>Streaming
102</a>
-are an excellent place to get started. A key takeaway from
-these articles: in realistic streaming scenarios with intermittent failures and
-disconnected users, data can arrive out of order or be delayed. Beam’s
-primitives provide a way for users to perform useful, powerful, and correct
-computations in spite of these challenges.</p>
-<p>As Beam pipeline authors, we need comprehensive tests that cover crucial
-failure scenarios and corner cases to gain real confidence that a pipeline is
-ready for production. The existing testing infrastructure within the Beam SDKs
-permits tests to be written which examine the contents of a Pipeline at
-execution time. However, writing unit tests for pipelines that may receive
-late data or trigger multiple times has historically ranged from complex to
-not possible, as pipelines that read from unbounded sources do not shut down
-without external intervention, while pipelines that read from bounded sources
-exclusively cannot test behavior with late data nor most speculative triggers.
-Without additional tools, pipelines that use custom triggers and handle
-out-of-order data could not be easily tested.</p>
-<p>This blog post introduces our new framework for writing tests for
pipelines that
-handle delayed and out-of-order data in the context of the LeaderBoard pipeline
-from the Mobile Gaming example series.</p>
-<h2 id="leaderboard-and-the-mobile-gaming-example">LeaderBoard and the
Mobile Gaming Example</h2>
-<p><a
href="https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java#L177";>LeaderBoard</a>
-is part of the <a
href="https://github.com/apache/beam/tree/master/examples/java/src/main/java/org/apache/beam/examples/complete/game";>Beam
mobile gaming examples</a>
-(and <a href="/get-started/mobile-gaming-example/">walkthroughs</a>)
-which produces a continuous accounting of user and team scores. User scores are
-calculated over the lifetime of the program, while team scores are calculated
-within fixed windows with a default duration of one hour. The LeaderBoard
-pipeline produces speculative and late panes as appropriate, based on the
-configured triggering and allowed lateness of the pipeline. The expected
outputs
-of the LeaderBoard pipeline vary depending on when elements arrive in relation
-to the watermark and the progress of processing time, which could not
previously
-be controlled within a test.</p>
-<h2 id="writing-deterministic-tests-to-emulate-nondeterminism">Writing
Deterministic Tests to Emulate Nondeterminism</h2>
-<p>The Beam testing infrastructure provides the
-<a
href="https://beam.apache.org/releases/javadoc/2.26.0/org/apache/beam/sdk/testing/PAssert.html";>PAssert</a>
-methods, which assert properties about the contents of a PCollection from
within
-a pipeline. We have expanded this infrastructure to include
-<a
href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java";>TestStream</a>,
-which is a PTransform that performs a series of events, consisting of adding
-additional elements to a pipeline, advancing the watermark of the TestStream,
-and advancing the pipeline processing time clock. TestStream permits tests
which
-observe the effects of triggers on the output a pipeline produces.</p>
-<p>While executing a pipeline that reads from a TestStream, the read waits
for all
-of the consequences of each event to complete before continuing on to the next
-event, ensuring that when processing time advances, triggers that are based on
-processing time fire as appropriate. With this transform, the effect of
-triggering and allowed lateness can be observed on a pipeline, including
-reactions to speculative and late panes and dropped data.</p>
-<h2 id="element-timings">Element Timings</h2>
-<p>Elements arrive either before, with, or after the watermark, which
categorizes
-them into the &ldquo;early&rdquo;, &ldquo;on-time&rdquo;, and
&ldquo;late&rdquo; divisions. &ldquo;Late&rdquo; elements can be
-further subdivided into &ldquo;unobservably&rdquo;,
&ldquo;observably&rdquo;, and &ldquo;droppably&rdquo; late,
-depending on the window to which they are assigned and the maximum allowed
-lateness, as specified by the windowing strategy. Elements that arrive with
-these timings are emitted into panes, which can be
&ldquo;EARLY&rdquo;, &ldquo;ON-TIME&rdquo;, or
-&ldquo;LATE&rdquo;, depending on the position of the watermark when
the pane was emitted.</p>
-<p>Using TestStream, we can write tests that demonstrate that speculative
panes are
-output after their trigger condition is met, that the advancing of the
watermark
-causes the on-time pane to be produced, and that late-arriving data produces
-refinements when it arrives before the maximum allowed lateness, and is dropped
-after.</p>
-<p>The following examples demonstrate how you can use TestStream to provide
a
-sequence of events to the Pipeline, where the arrival of elements is
interspersed
-with updates to the watermark and the advance of processing time. Each of these
-events runs to completion before additional events occur.</p>
-<p>In the diagrams, the time at which events occurred in
&ldquo;real&rdquo; (event) time
-progresses as the graph moves to the right. The time at which the pipeline
-receives them progresses as the graph goes upwards. The watermark is
represented
-by the squiggly red line, and each starburst is the firing of a trigger and the
-associated pane.</p>
-<img class="center-block"
src="/images/blog/test-stream/elements-all-on-time.png" alt="Elements on the
Event and Processing time axes, with the Watermark and produced panes"
width="442">
-<h3 id="everything-arrives-on-time">Everything arrives on-time</h3>
-<p>For example, if we create a TestStream where all the data arrives before
the
-watermark and provide the result PCollection as input to the
CalculateTeamScores
-PTransform:</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant& [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class= [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class= [...]
- <span class="c1">// Move the watermark past the end the end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<p>we can then assert that the result PCollection contains elements that
arrived:</p>
-<img class="center-block"
src="/images/blog/test-stream/elements-all-on-time.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// Only one value is
emitted for the blue team
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="some-elements-are-late-but-arrive-before-the-end-of-the-window">Some
elements are late, but arrive before the end of the window</h3>
-<p>We can also add data to the TestStream after the watermark, but before
the end
-of the window (shown below to the left of the red watermark), which
demonstrates
-&ldquo;unobservably late&rdquo; data - that is, data that arrives
late, but is promoted by
-the system to be on time, as it arrives before the watermark passes the end of
-the window</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant&l [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).&l [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-unobservably-late.png" alt="An element
arrives late, but before the watermark passes the end of the window, and is
produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// Only one value is
emitted for the blue team
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3 id="elements-are-late-and-arrive-after-the-end-of-the-window">Elements
are late, and arrive after the end of the window</h3>
-<p>By advancing the watermark farther in time before adding the late data,
we can
-demonstrate the triggering behavior that causes the system to emit an on-time
-pane, and then after the late data arrives, a pane that refines the
result.</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="k">new</span> <span class="n">Instant&l [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">). [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">12</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-observably-late.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// An on-time pane
is emitted with the events that arrived before the window closed
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inOnTimePane</span><span class="o">(</span><span
class="n">window</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">6</span><span class="o">));</span>
-<span class="c1">// The final pane contains the late refinement
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inFinalPane</span><span class="o">(</span><span
class="n">window</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">18</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="elements-are-late-and-after-the-end-of-the-window-plus-the-allowed-lateness">Elements
are late, and after the end of the window plus the allowed lateness</h3>
-<p>If we push the watermark even further into the future, beyond the maximum
-configured allowed lateness, we can demonstrate that the late element is
dropped
-by the system.</p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">&lt;</span><span
class="n">GameActionInfo</span><span class="o">&gt;</span>
<span class="n">infos</span> <span class="o">=</span> <span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span [...]
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span class="s">&#34;sky&#34;</span><span
class="o">,</span> <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">3</span><span class="o">,</span> <span
class="n">Duration</span><span class="o">.< [...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;navy&#34;</span><span class="o">,</span>
<span class="s">&#34;blue&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span c [...]
- <span class="c1">// Move the watermark up to &#34;near&#34; the
end of the window
-</span><span class="c1"></span> <span
class="o">.</span><span class="na">advanceWatermarkTo</span><span
class="o">(</span><span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">ALLOWED_LATENESS</span><span class="o">)</span>
- <span class="o">.</span><span
class="na">plus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes</span><span class="o">(</span><span
class="n">1</span><span class="o">)))</span>
-<span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span>
- <span
class="s">&#34;sky&#34;</span><span class="o">,</span>
- <span
class="s">&#34;blue&#34;</span><span class="o">,</span>
- <span class="n">12</span><span
class="o">,</span>
- <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0</span><span class="o">).</span><span
class="na">plus</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">).</span><span
class="na">minus</span><span class="o">(</span><span
class="n">Duration</span><span class="o">.</span><span
class="na">standardMinutes [...]
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">teamScores</span>
<span class="o">=</span> <span class="n">p</span><span
class="o">.</span><span class="na">apply</span><span
class="o">(</span>< [...]
-<span class="o">.</span><span class="na">apply</span><span
class="o">(</span><span class="k">new</span> <span
class="n">CalculateTeamScores</span><span class="o">(</span><span
class="n">TEAM_WINDOW_DURATION</span><span class="o">,</span> <span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-droppably-late.png" alt="Elements all
arrive before the watermark, and are produced in the on-time pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span class="c1">// An on-time pane
is emitted with the events that arrived before the window closed
-</span><span class="c1"></span><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">teamScores</span><span class="o">)</span>
-<span class="o">.</span><span class="na">inWindow</span><span
class="o">(</span><span class="n">window</span><span
class="o">)</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;blue&#34;</span><span class="o">,</span>
<span class="n">6</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h3
id="elements-arrive-before-the-end-of-the-window-and-some-processing-time-passes">Elements
arrive before the end of the window, and some processing time passes</h3>
-<p>Using additional methods, we can demonstrate the behavior of speculative
-triggers by advancing the processing time of the TestStream. If we add elements
-to an input PCollection, occasionally advancing the processing time clock, and
-apply <code>CalculateUserScores</code></p>
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">TestStream</span><span class="o">.</span><span
class="na">create</span><span class="o">(</span><span
class="n">AvroCoder</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="n">GameActionInfo</span><span class="o">.</span><span
class="na">class</span><span class="o">))& [...]
- <span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">3</span><span
class="o">,</span> <span class="k">new</span> <span class="n">Ins
[...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">2</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class [...]
-<span class="o">.</span><span
class="na">advanceProcessingTime</span><span
class="o">(</span><span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span class="n">12</span><span
class="o">))</span>
- <span class="o">.</span><span
class="na">addElements</span><span class="o">(</span><span
class="k">new</span> <span class="n">GameActionInfo</span><span
class="o">(</span><span
class="s">&#34;oxblood&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">2</span><span
class="o">,</span> <span class="k">new</span> <span class="n">Ins
[...]
- <span class="k">new</span> <span
class="n">GameActionInfo</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="s">&#34;red&#34;</span><span
class="o">,</span> <span class="n">4</span><span
class="o">,</span> <span class="k">new</span> <span
class="n">Instant</span><span class="o">(</span><span
class="n">0L</span><span class [...]
-<span class="o">.</span><span
class="na">advanceProcessingTime</span><span
class="o">(</span><span class="n">Duration</span><span
class="o">.</span><span class="na">standardMinutes</span><span
class="o">(</span><span class="n">15</span><span
class="o">))</span>
-<span class="o">.</span><span
class="na">advanceWatermarkToInfinity</span><span class="o">();</span>
-<span class="n">PCollection</span><span
class="o">&lt;</span><span class="n">KV</span><span
class="o">&lt;</span><span class="n">String</span><span
class="o">,</span> <span class="n">Integer</span><span
class="o">&gt;&gt;</span> <span class="n">userScores</span>
<span class="o">=</span>
- <span class="n">p</span><span class="o">.</span><span
class="na">apply</span><span class="o">(</span><span
class="n">infos</span><span class="o">).</span><span
class="na">apply</span><span class="o">(</span><span
class="k">new</span> <span
class="n">CalculateUserScores</span><span class="o">(</span><span
class="n">ALLOWED_LATENESS</span><span
class="o">));</span></code></pre></div>
-</div>
-<img class="center-block"
src="/images/blog/test-stream/elements-processing-speculative.png"
alt="Elements all arrive before the watermark, and are produced in the on-time
pane" width="442">
-<div class=language-java>
-<div class="highlight"><pre class="chroma"><code
class="language-java" data-lang="java"><span
class="n">PAssert</span><span class="o">.</span><span
class="na">that</span><span class="o">(</span><span
class="n">userScores</span><span class="o">)</span>
-<span class="o">.</span><span
class="na">inEarlyGlobalWindowPanes</span><span class="o">()</span>
-<span class="o">.</span><span
class="na">containsInAnyOrder</span><span class="o">(</span><span
class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="n">5</span><span class="o">),</span>
- <span class="n">KV</span><span
class="o">.</span><span class="na">of</span><span
class="o">(</span><span
class="s">&#34;scarlet&#34;</span><span class="o">,</span>
<span class="n">9</span><span class="o">),</span>
-<span class="n">KV</span><span class="o">.</span><span
class="na">of</span><span class="o">(</span><span
class="s">&#34;oxblood&#34;</span><span class="o">,</span>
<span class="n">2</span><span class="o">));</span>
-<span class="n">p</span><span class="o">.</span><span
class="na">run</span><span
class="o">();</span></code></pre></div>
-</div>
-<h2 id="teststream---under-the-hood">TestStream - Under the Hood</h2>
-<p>TestStream relies on a pipeline concept we’ve introduced, called
quiescence, to
-utilize the existing runner infrastructure while providing guarantees about
when
-a root transform will called by the runner. This consists of properties about
-pending elements and triggers, namely:</p>
-<ul>
-<li>No trigger is permitted to fire but has not fired</li>
-<li>All elements are either buffered in state or cannot progress until a
side input becomes available</li>
-</ul>
-<p>Simplified, this means that, in the absence of an advancement in input
-watermarks or processing time, or additional elements being added to the
-pipeline, the pipeline will not make progress. Whenever the TestStream
PTransform
-performs an action, the runner must not reinvoke the same instance until the
-pipeline has quiesced. This ensures that the events specified by TestStream
-happen &ldquo;in-order&rdquo;, which ensures that input watermarks and
the system clock do
-not advance ahead of the elements they hoped to hold up.</p>
-<p>The DirectRunner has been modified to use quiescence as the signal that
it
-should add more work to the Pipeline, and the implementation of TestStream in
-that runner uses this fact to perform a single output per event. The
DirectRunner
-implementation also directly controls the runner’s system clock, ensuring that
-tests will complete promptly even if there is a multi-minute processing time
-trigger located within the pipeline.</p>
-<p>The TestStream transform is supported in the DirectRunner. For most
users, tests
-written using TestPipeline and PAsserts will automatically function while using
-TestStream.</p>
-<h2 id="summary">Summary</h2>
-<p>The addition of TestStream alongside window and pane-specific matchers
in PAssert
-has enabled the testing of Pipelines which produce speculative and late panes.
-This permits tests for all styles of pipeline to be expressed directly within
the
-Java SDK. If you have questions or comments, we’d love to hear them on the
-<a href="/get-started/support/">mailing
lists</a>.</p></description></item></channel></rss>
\ No newline at end of file
+<p>The next step is to take the Apex runner from functional to ready for
real applications that run distributed, leveraging the scalability and
performance features of Apex, similar to its native API. This includes chaining
of ParDos, partitioning, optimizing combine operations etc. To get involved,
please see <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20and%20component%20%3D%20runner-apex%20and%20resolution%20%3D%20unresolved";>JIRA</a>
and join the [...]
\ No newline at end of file
diff --git a/website/generated-content/categories/index.xml
b/website/generated-content/categories/index.xml
index cce5cd9..77e0d41 100644
--- a/website/generated-content/categories/index.xml
+++ b/website/generated-content/categories/index.xml
@@ -1 +1 @@
-<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
Categories</title><link>/categories/</link><description>Recent content in
Categories on Apache Beam</description><generator>Hugo --
gohugo.io</generator><lastBuildDate>Fri, 11 Dec 2020 12:00:00
-0800</lastBuildDate><atom:link href="/categories/index.xml" rel="self"
type="application/rss+xml"/></channel></rss>
\ No newline at end of file
+<rss version="2.0"
xmlns:atom="http://www.w3.org/2005/Atom";><channel><title>Apache Beam –
Categories</title><link>/categories/</link><description>Recent content in
Categories on Apache Beam</description><generator>Hugo --
gohugo.io</generator><lastBuildDate>Mon, 14 Dec 2020 00:00:01
-0800</lastBuildDate><atom:link href="/categories/index.xml" rel="self"
type="application/rss+xml"/></channel></rss>
\ No newline at end of file
diff --git a/website/generated-content/feed.xml
b/website/generated-content/feed.xml
index 85db611..9d3ee53 100644
--- a/website/generated-content/feed.xml
+++ b/website/generated-content/feed.xml
@@ -1,4 +1,86 @@
-<rss xmlns:atom="http://www.w3.org/2005/Atom";
version="2.0"><channel><title>Apache Beam</title><description>Apache Beam is an
open source, unified model and set of language-specific SDKs for defining and
executing data processing workflows, and also data ingestion and integration
flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific
Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale
batch and streaming data processing and can run on a number [...]
+<rss xmlns:atom="http://www.w3.org/2005/Atom";
version="2.0"><channel><title>Apache Beam</title><description>Apache Beam is an
open source, unified model and set of language-specific SDKs for defining and
executing data processing workflows, and also data ingestion and integration
flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific
Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale
batch and streaming data processing and can run on a number [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>We are pleased to announce that Splittable DoFn (SDF) is ready for use
in the Beam Python, Java,
+and Go SDKs for versions 2.25.0 and later.</p>
+<p>In 2017, <a
href="https://beam.apache.org/blog/splittable-do-fn/";>Splittable DoFn Blog
Post</a> proposed
+to build <a href="https://s.apache.org/splittable-do-fn";>Splittable
DoFn</a> APIs as the new recommended way of
+building I/O connectors. Splittable DoFn is a generalization of
<code>DoFn</code> that gives it the core
+capabilities of <code>Source</code> while retaining
<code>DoFn</code>'s syntax, flexibility, modularity, and ease of
+coding. Thus, it becomes much easier to develop complex I/O connectors with
simpler and reusable
+code.</p>
+<p>SDF has three advantages over the existing
<code>UnboundedSource</code> and <code>BoundedSource</code>:</p>
+<ul>
+<li>SDF provides a unified set of APIs to handle both unbounded and bounded
cases.</li>
+<li>SDF enables reading from source descriptors dynamically.
+<ul>
+<li>Taking KafkaIO as an example, within
<code>UnboundedSource</code>/<code>BoundedSource</code> API, you
must specify
+the topic and partition you want to read from during pipeline construction
time. There is no way
+for <code>UnboundedSource</code>/<code>BoundedSource</code> to
accept topics and partitions as inputs during execution
+time. But it&rsquo;s built-in to SDF.</li>
+</ul>
+</li>
+<li>SDF fits in as any node on a pipeline freely with the ability of
splitting.
+<ul>
+<li><code>UnboundedSource</code>/<code>BoundedSource</code> has
to be the root node of the pipeline to gain performance
+benefits from splitting strategies, which limits many real-world usages. This
is no longer a limit
+for an SDF.</li>
+</ul>
+</li>
+</ul>
+<p>As SDF is now ready to use with all the mentioned improvements, it is
the recommended
+way to build the new I/O connectors. Try out building your own Splittable DoFn
by following the
+<a
href="https://beam.apache.org/documentation/programming-guide/#splittable-dofns";>programming
guide</a>. We
+have provided tonnes of common utility classes such as common types of
<code>RestrictionTracker</code> and
+<code>WatermarkEstimator</code> in Beam SDK, which will help you onboard
easily. As for the existing I/O
+connectors, we have wrapped <code>UnboundedSource</code> and
<code>BoundedSource</code> implementations into Splittable
+DoFns, yet we still encourage developers to convert
<code>UnboundedSource</code>/<code>BoundedSource</code> into actual
+Splittable DoFn implementation to gain more performance benefits.</p>
+<p>Many thanks to every contributor who brought this highly anticipated
design into the data processing
+world. We are really excited to see that users benefit from SDF.</p>
+<p>Below are some real-world SDF examples for you to explore.</p>
+<h2 id="real-world-splittable-dofn-examples">Real world Splittable DoFn
examples</h2>
+<p><strong>Java Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java#L118";>Kafka</a>:
+An I/O connector for <a href="https://kafka.apache.org/";>Apache Kafka</a>
+(an open-source distributed event streaming platform).</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java#L787";>Watch</a>:
+Uses a polling function producing a growing set of outputs for each input
until a per-input
+termination condition is met.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java#L365";>Parquet</a>:
+An I/O connector for <a href="https://parquet.apache.org/";>Apache
Parquet</a>
+(an open-source columnar storage format).</li>
+<li><a
href="https://github.com/apache/beam/blob/6fdde4f4eab72b49b10a8bb1cb3be263c5c416b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java#L493";>HL7v2</a>:
+An I/O connector for HL7v2 messages (a clinical messaging format that provides
data about events
+that occur inside an organization) part of
+<a href="https://cloud.google.com/healthcare";>Google’s Cloud Healthcare
API</a>.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L248";>BoundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L432";>UnboundedSource
wrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html";>UnboundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Python Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/python/apache_beam/io/iobase.py#L1375";>BoundedSourceWrapper</a>:
+A wrapper which converts an existing <a
href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.iobase.html#apache_beam.io.iobase.BoundedSource";>BoundedSource</a>
+implementation to a splittable DoFn.</li>
+</ul>
+<p><strong>Go Examples</strong></p>
+<ul>
+<li><a
href="https://github.com/apache/beam/blob/ce190e11332469ea59b6c9acf16ee7c673ccefdd/sdks/go/pkg/beam/io/textio/sdf.go#L40";>textio.ReadSdf</a>
implements reading from text files using a splittable DoFn.</li>
+</ul></description><link>/blog/splittable-do-fn-is-available/</link><pubDate>Mon,
14 Dec 2020 00:00:01
-0800</pubDate><guid>/blog/splittable-do-fn-is-available/</guid><category>blog</category></item><item><title>Apache
Beam 2.26.0</title><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -763,102 +845,4 @@ Kotlin a try.</p>
<p>I&rsquo;d like to extend a very special thanks to <a
href="https://twitter.com/henry_ken";>Henry Suryawirawan</a> for his creation
of the original series of Katas
and his support during the review process and making this effort a
reality.</p>
<p><br /></p>
-<p><img
src="/images/blog/beam-katas-kotlin-release/beam-katas-in-edutools.png"
alt="Access Beam Katas Kotlin through a JetBrains Educational Product"
height="252" width="800"
></p></description><link>/blog/beam-katas-kotlin-release/</link><pubDate>Mon,
01 Jun 2020 00:00:01
-0800</pubDate><guid>/blog/beam-katas-kotlin-release/</guid><category>blog</category></item><item><title>Python
SDK Typing Changes</title><description>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>Beam Python has recently increased its support and integration of Python
3 type
-annotations for improved code clarity and type correctness checks.
-Read on to find out what&rsquo;s new.</p>
-<p>Python supports type annotations on functions (PEP 484). Static type
checkers,
-such as mypy, are used to verify adherence to these types.
-For example:</p>
-<pre><code>def f(v: int) -&gt; int:
-return v[0]
-</code></pre><p>Running mypy on the above code will give the error:
-<code>Value of type &quot;int&quot; is not
indexable</code>.</p>
-<p>We&rsquo;ve recently made changes to Beam in 2 areas:</p>
-<p>Adding type annotations throughout Beam. Type annotations make a large
and
-sophisticated codebase like Beam easier to comprehend and navigate in your
-favorite IDE.</p>
-<p>Second, we&rsquo;ve added support for Python 3 type annotations.
This allows SDK
-users to specify a DoFn&rsquo;s type hints in one place.
-We&rsquo;ve also expanded Beam&rsquo;s support of
<code>typing</code> module types.</p>
-<p>For more background see:
-<a
href="https://beam.apache.org/documentation/sdks/python-type-safety/";>Ensuring
Python Type Safety</a>.</p>
-<h1 id="beam-is-typed">Beam Is Typed</h1>
-<p>In tandem with the new type annotation support within DoFns,
we&rsquo;ve invested a
-great deal of time adding type annotations to the Beam python code itself.
-With this in place, we have begun using mypy, a static type
-checker, as part of Beam&rsquo;s code review process, which ensures higher
quality
-contributions and fewer bugs.
-The added context and insight that type annotations add throughout Beam is
-useful for all Beam developers, contributors and end users alike, but
-it is especially beneficial for developers who are new to the project.
-If you use an IDE that understands type annotations, it will provide richer
-type completions and warnings than before.
-You&rsquo;ll also be able to use your IDE to inspect the types of Beam
functions and
-transforms to better understand how they work, which will ease your own
-development.
-Finally, once Beam is fully annotated, end users will be able to benefit from
-the use of static type analysis on their own pipelines and custom
transforms.</p>
-<h1 id="new-ways-to-annotate">New Ways to Annotate</h1>
-<h2 id="python-3-syntax-annotations">Python 3 Syntax Annotations</h2>
-<p>Coming in Beam 2.21 (BEAM-8280), you will be able to use Python
annotation
-syntax to specify input and output types.</p>
-<p>For example, this new form:</p>
-<pre><code>class MyDoFn(beam.DoFn):
-def process(self, element: int) -&gt; typing.Text:
-yield str(element)
-</code></pre><p>is equivalent to this:</p>
-<pre><code>@apache_beam.typehints.with_input_types(int)
-@apache_beam.typehints.with_output_types(typing.Text)
-class MyDoFn(beam.DoFn):
-def process(self, element):
-yield str(element)
-</code></pre><p>One of the advantages of the new form is that you may
already be using it
-in tandem with a static type checker such as mypy, thus getting additional
-runtime type checking for free.</p>
-<p>This feature will be enabled by default, and there will be 2 mechanisms
in
-place to disable it:</p>
-<ol>
-<li>Calling
<code>apache_beam.typehints.disable_type_annotations()</code> before
pipeline
-construction will disable the new feature completely.</li>
-<li>Decorating a function with
<code>@apache_beam.typehints.no_annotations</code> will
-tell Beam to ignore annotations for it.</li>
-</ol>
-<p>Uses of Beam&rsquo;s <code>with_input_type</code>,
<code>with_output_type</code> methods and decorators will
-still work and take precedence over annotations.</p>
-<h3 id="sidebar">Sidebar</h3>
-<p>You might ask: couldn&rsquo;t we use mypy to type check Beam
pipelines?
-There are several reasons why this is not the case.</p>
-<ul>
-<li>Pipelines are constructed at runtime and may depend on information that
is
-only known at that time, such as a config file or database table
schema.</li>
-<li>PCollections don&rsquo;t have the necessary type information, so
mypy sees them as
-effectively containing any element type.
-This may change in in the future.</li>
-<li>Transforms using lambdas (ex: <code>beam.Map(lambda x: (1,
x)</code>) cannot be
-annotated properly using PEP 484.
-However, Beam does a best-effort attempt to analyze the output type
-from the bytecode.</li>
-</ul>
-<h2 id="typing-module-support">Typing Module Support</h2>
-<p>Python&rsquo;s <a
href="https://docs.python.org/3/library/typing.html";>typing</a> module
defines
-types used in type annotations. This is what we call
&ldquo;native&rdquo; types.
-While Beam has its own typing types, it also supports native types.
-While both Beam and native types are supported, for new code we encourage using
-native typing types. Native types have as these are supported by additional
tools.</p>
-<p>While working on Python 3 annotations syntax support, we&rsquo;ve
also discovered and
-fixed issues with native type support. There may still be bugs and unsupported
-native types. Please
-<a href="https://beam.apache.org/community/contact-us/";>let us know</a>
if you encounter
-issues.</p></description><link>/blog/python-typing/</link><pubDate>Thu, 28
May 2020 00:00:01
-0800</pubDate><guid>/blog/python-typing/</guid><category>blog</category><category>python</category><category>typing</category></item></channel></rss>
\ No newline at end of file
+<p><img
src="/images/blog/beam-katas-kotlin-release/beam-katas-in-edutools.png"
alt="Access Beam Katas Kotlin through a JetBrains Educational Product"
height="252" width="800"
></p></description><link>/blog/beam-katas-kotlin-release/</link><pubDate>Mon,
01 Jun 2020 00:00:01
-0800</pubDate><guid>/blog/beam-katas-kotlin-release/</guid><category>blog</category></item></channel></rss>
\ No newline at end of file
diff --git a/website/generated-content/index.html
b/website/generated-content/index.html
index 3ba331d..93e1aef 100644
--- a/website/generated-content/index.html
+++ b/website/generated-content/index.html
@@ -5,7 +5,7 @@
<a class="button button--primary" href=/get-started/try-apache-beam/>Try
Beam</a>
<a class="button button--primary" href=/get-started/downloads/>Download Beam
SDK 2.26.0</a></div><div class=hero__ctas><a class=button
href=/get-started/quickstart-java/>Java Quickstart</a>
<a class=button href=/get-started/quickstart-py/>Python Quickstart</a>
-<a class=button href=/get-started/quickstart-go/>Go
Quickstart</a></div></div></div><div class=hero__cols__col><div
class=hero__blog><div class=hero__blog__title>The latest from the
blog</div><div class=hero__blog__cards><a class=hero__blog__cards__card
href=/blog/beam-2.26.0/><div class=hero__blog__cards__card__title>Apache Beam
2.26.0</div><div class=hero__blog__cards__card__date>Dec 11, 2020</div></a><a
class=hero__blog__cards__card href=/blog/beam-2.25.0/><div
class=hero__blog__cards [...]
+<a class=button href=/get-started/quickstart-go/>Go
Quickstart</a></div></div></div><div class=hero__cols__col><div
class=hero__blog><div class=hero__blog__title>The latest from the
blog</div><div class=hero__blog__cards><a class=hero__blog__cards__card
href=/blog/splittable-do-fn-is-available/><div
class=hero__blog__cards__card__title>Splittable DoFn in Apache Beam is Ready to
Use</div><div class=hero__blog__cards__card__date>Dec 14, 2020</div></a><a
class=hero__blog__cards__card href=/ [...]
<a class="button button--primary" href=/get-started/downloads/>Download Beam
SDK 2.26.0</a></div><div class=ctas__ctas><a class=button
href=/get-started/quickstart-java/>Java Quickstart</a>
<a class=button href=/get-started/quickstart-py/>Python Quickstart</a>
<a class=button href=/get-started/quickstart-go/>Go
Quickstart</a></div></div></div><footer class=footer><div
class=footer__contained><div class=footer__cols><div
class=footer__cols__col><div class=footer__cols__col__logo><img
src=/images/beam_logo_circle.svg class=footer__logo alt="Beam logo"></div><div
class=footer__cols__col__logo><img src=/images/apache_logo_circle.svg
class=footer__logo alt="Apache logo"></div></div><div class="footer__cols__col
footer__cols__col--md"><div class=foo [...]
diff --git a/website/generated-content/sitemap.xml
b/website/generated-content/sitemap.xml
index b4ee870..fd3236e 100644
--- a/website/generated-content/sitemap.xml
+++ b/website/generated-content/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset
xmlns="http://www.sitemaps.org/schemas/sitemap/0.9";
xmlns:xhtml="http://www.w3.org/1999/xhtml";><url><loc>/blog/beam-2.26.0/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/categories/blog/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/blog/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/categories/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/blog/b
[...]
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset
xmlns="http://www.sitemaps.org/schemas/sitemap/0.9";
xmlns:xhtml="http://www.w3.org/1999/xhtml";><url><loc>/categories/blog/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/blog/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/categories/</loc><lastmod>2020-12-11T13:21:47-08:00</lastmod></url><url><loc>/blog/splittable-do-fn-is-available/</loc><lastmod>2020-12-01T17:42:26-08:00</lastmod></url
[...]
\ No newline at end of file
