[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/dd65b3b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/dd65b3b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/dd65b3b6 Branch: refs/heads/asf-site Commit: dd65b3b6c769cc23ea2c901aca2219bfd5e32c40 Parents: c54a9df Author: Davor Bonaci Authored: Mon Nov 7 10:20:35 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 7 10:20:35 2016 -0800 -- .../2016/03/17/capability-matrix.html | 1 + .../2016/04/03/presentation-materials.html | 1 + .../sdk/2016/02/25/python-sdk-now-public.html | 1 + .../beam/release/2016/06/15/first-release.html | 1 + .../10/11/strata-hadoop-world-and-beam.html | 1 + .../website/2016/02/22/beam-has-a-logo.html | 1 + .../blog/2016/05/18/splitAtFraction-method.html | 1 + .../05/27/where-is-my-pcollection-dot-map.html | 1 + .../06/13/flink-batch-runner-milestone.html | 1 + content/blog/2016/08/03/six-months.html | 1 + content/blog/2016/10/20/test-stream.html| 1 + content/blog/index.html | 1 + content/coming-soon.html| 1 + .../contribute/contribution-guide/index.html| 1 + content/contribute/design-principles/index.html | 1 + content/contribute/index.html | 1 + content/contribute/logos/index.html | 1 + .../presentation-materials/index.html | 1 + content/contribute/release-guide/index.html | 1 + content/contribute/source-repository/index.html | 1 + content/contribute/team/index.html | 1 + content/contribute/testing/index.html | 1 + content/contribute/work-in-progress/index.html | 1 + content/documentation/index.html| 3 +- .../pipelines/create-your-pipeline/index.html | 1 + .../pipelines/design-your-pipeline/index.html | 1 + .../pipelines/test-your-pipeline/index.html | 1 + .../documentation/programming-guide/index.html | 1 + content/documentation/resources/index.html | 1 + .../runners/capability-matrix/index.html| 289 ++- .../documentation/runners/dataflow/index.html | 1 + content/documentation/runners/direct/index.html | 1 + content/documentation/runners/flink/index.html | 1 + content/documentation/runners/spark/index.html | 1 + content/documentation/sdks/java/index.html | 1 + content/documentation/sdks/javadoc/index.html | 1 + content/get-started/beam-overview/index.html| 1 + content/get-started/downloads/index.html| 1 + content/get-started/index.html | 1 + .../mobile-gaming-example/index.html| 1 + content/get-started/quickstart/index.html | 1 + content/get-started/releases/index.html | 1 + content/get-started/support/index.html | 1 + .../get-started/wordcount-example/index.html| 1 + content/index.html | 1 + content/privacy_policy/index.html | 1 + 46 files changed, 334 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/dd65b3b6/content/beam/capability/2016/03/17/capability-matrix.html -- diff --git a/content/beam/capability/2016/03/17/capability-matrix.html b/content/beam/capability/2016/03/17/capability-matrix.html index d9ec070..405c68b 100644 --- a/content/beam/capability/2016/03/17/capability-matrix.html +++ b/content/beam/capability/2016/03/17/capability-matrix.html @@ -84,6 +84,7 @@ Runners Capability Matrix Direct Runner + Apache Apex Runner Apache Flink Runner Apache Spark Runner Cloud Dataflow Runner http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/dd65b3b6/content/beam/capability/2016/04/03/presentation-materials.html -- diff --git a/content/beam/capability/2016/04/03/presentation-materials.html b/content/beam/capability/2016/04/03/presentation-materials.html index 550f4d7..766c6d3 100644 --- a/content/beam/capability/2016/04/03/presentation-materials.html +++ b/content/beam/capability/2016/04/03/presentation-materials.html @@ -84,6 +84,7 @@ Runners Capability Matrix Direct Runner + Apache Apex Runner Apache Flink Runner Apache S
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/f941c023 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/f941c023 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/f941c023 Branch: refs/heads/asf-site Commit: f941c0230c94f0bd541057dbd5cc68a0b08bad3f Parents: 2fc6f45 Author: Davor Bonaci Authored: Mon Nov 7 12:21:15 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 7 12:21:15 2016 -0800 -- content/documentation/runners/apex/index.html | 168 content/get-started/quickstart/index.html | 105 - content/js/language-switch.js | 173 +++-- 3 files changed, 292 insertions(+), 154 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/f941c023/content/documentation/runners/apex/index.html -- diff --git a/content/documentation/runners/apex/index.html b/content/documentation/runners/apex/index.html new file mode 100644 index 000..f25dec0 --- /dev/null +++ b/content/documentation/runners/apex/index.html @@ -0,0 +1,168 @@ + + + + + + + + + Apache Apex Runner + + + + + https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";> + + + http://beam.incubator.apache.org/documentation/runners/apex/"; data-proofer-ignore> + http://beam.incubator.apache.org/feed.xml";> + + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + +ga('create', 'UA-73650088-1', 'auto'); +ga('send', 'pageview'); + + + + + + + + + + + + + + + +Toggle navigation + + + + + + + + + Get Started + + Beam Overview + Quickstart + + Example Walkthroughs + WordCount + Mobile Gaming + + Resources + Downloads + Release Notes + Support + + + + Documentation + + Using the Documentation + + Beam Concepts + Programming Guide + Additional Resources + + Pipeline Fundamentals + Design Your Pipeline + Create Your Pipeline + Test Your Pipeline + + SDKs + Java SDK + Java SDK API Reference + + Runners + Capability Matrix + Direct Runner + Apache Apex Runner + Apache Flink Runner + Apache Spark Runner + Cloud Dataflow Runner + + + + Contribute + + Get Started Contributing + +Guides + Contribution Guide +Testing Guide +Release Guide + +Technical References +Design Principles + Ongoing Projects +Source Repository + + Promotion +Presentation Materials +Logos and Design + +Team + + + +Blog + + + + https://www.apache.org/foundation/press/kit/feather_small.png"; alt="Apache Logo" style="height:24px;">Apache Software Foundation + +http://www.apache.org/";>ASF Homepage +http://www.apache.org/licenses/";>License +http://www.apache.org/security/";>Security +http://www.apache.org/foundation/thanks.html";>Thanks +http://www.apache.org/foundation/sponsorship.html";>Sponsorship +https://www.apache.org/foundation/policies/conduct";>Code of Conduct + + + + + + + + + + + + + + +Using the Apache Apex Runner + +This page is under construction (https://issues.apache.org/jira/browse/BEAM-825";>BEAM-825). The runner is on a feature branch. + + + + + +
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/37562256 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/37562256 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/37562256 Branch: refs/heads/asf-site Commit: 37562256e41a66a90171a471deb88a7e741872bb Parents: b3c3c88 Author: Davor Bonaci Authored: Tue Nov 8 15:58:25 2016 -0800 Committer: Davor Bonaci Committed: Tue Nov 8 15:58:25 2016 -0800 -- content/contribute/team/index.html | 14 +++--- 1 file changed, 7 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/37562256/content/contribute/team/index.html -- diff --git a/content/contribute/team/index.html b/content/contribute/team/index.html index e5e420a..2c8fd04 100644 --- a/content/contribute/team/index.html +++ b/content/contribute/team/index.html @@ -244,8 +244,8 @@ Jesse Anderson - - jesse [at] smokinghand [dot] com + jesseanderson + jesseanderson [at] apache [dot] org Smoking Hand committer -8 @@ -263,7 +263,7 @@ Kostas Tzoumas ktzoumas - kostas [at] data-artisans [dot] com + ktzoumas [at] apache [dot] org data Artisans committer, PPMC +1 @@ -299,7 +299,7 @@ Stephan Ewen sewen - stephan [at] data-artisans [dot] com + sewen [at] apache [dot] org data Artisans committer, PPMC +1 @@ -307,8 +307,8 @@ Thomas Groh - - + tgroh + tgroh [at] apache [dot] org Google committer -8 @@ -326,7 +326,7 @@ Tom White tomwhite - tom [at] cloudera [dot] com + tomwhite [at] apache [dot] org Cloudera committer, PPMC -8
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/13d13604 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/13d13604 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/13d13604 Branch: refs/heads/asf-site Commit: 13d1360478855ac5e8386ddc60b0ca510bf03572 Parents: b372fd9 Author: Davor Bonaci Authored: Sun Nov 13 16:20:05 2016 -0800 Committer: Davor Bonaci Committed: Sun Nov 13 16:20:05 2016 -0800 -- content/documentation/runners/apex/index.html | 2 +- .../runners/capability-matrix/index.html| 16 +++ content/get-started/quickstart/index.html | 21 +++- content/index.html | 6 +- 4 files changed, 34 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/13d13604/content/documentation/runners/apex/index.html -- diff --git a/content/documentation/runners/apex/index.html b/content/documentation/runners/apex/index.html index f25dec0..fcbb722 100644 --- a/content/documentation/runners/apex/index.html +++ b/content/documentation/runners/apex/index.html @@ -142,7 +142,7 @@ Using the Apache Apex Runner -This page is under construction (https://issues.apache.org/jira/browse/BEAM-825";>BEAM-825). The runner is on a feature branch. +This page is under construction (https://issues.apache.org/jira/browse/BEAM-825";>BEAM-825). http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/13d13604/content/documentation/runners/capability-matrix/index.html -- diff --git a/content/documentation/runners/capability-matrix/index.html b/content/documentation/runners/capability-matrix/index.html index 30d5532..a5e4190 100644 --- a/content/documentation/runners/capability-matrix/index.html +++ b/content/documentation/runners/capability-matrix/index.html @@ -190,7 +190,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -482,7 +482,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -714,7 +714,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -976,7 +976,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -1102,7 +1102,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -1394,7 +1394,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -1626,7 +1626,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex @@ -1888,7 +1888,7 @@ Apache Spark -Apache Apex (on feature branch) +Apache Apex http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/13d13604/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index c4f6772..bc7396c 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -196,7 +196,7 @@ MinimalWordCount.java WordCount.java Run WordCount -A single Beam pipeline can run on multiple Beam runners, including the SparkRunner, FlinkRunner, or DataflowRunner. The DirectRunner is a common runner for getting started, as it runs locally on your machine and requires no specific setup. +A single Beam pipeline can run on multiple Beam runners, including the ApexRunner, FlinkRunner, SparkRunner or DataflowRunner. The DirectRunner is a common runner for getting started, as it runs locally on your machine and requires no specific setup. After youâve chosen which runner youâd like to use: @@ -217,6 +217,11 @@ MinimalWordCount.java WordCount.java +$ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \ + -Dexec.args="--inputFile=pom.xml --output=counts --runner=ApexRunner" -Papex-runner + + + TODO BEAM-899 @@ -239,6 +244,10 @@ MinimalWordCount.java WordCount.java +$ ls counts* + + + TODO BEAM-899 @@ -265,6 +274,16 @@ Foundation: 1 +$ cat counts* +BEAM: 1 +have: 1 +simple: 1 +skip: 4 +PAssert: 1 +... + + + TODO BEAM-899 http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/13d13604/content/index.html -- diff --git a/content/index.html b/content/index.html index 3ded4c5..84e835c 100644 --- a/content/
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/de57d1ae Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/de57d1ae Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/de57d1ae Branch: refs/heads/asf-site Commit: de57d1aefe42049a1ab07fc26f25632118bfddf0 Parents: 5411eed Author: Davor Bonaci Authored: Mon Nov 14 09:33:04 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 14 09:33:04 2016 -0800 -- content/blog/2016/10/20/test-stream.html | 2 +- content/feed.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/de57d1ae/content/blog/2016/10/20/test-stream.html -- diff --git a/content/blog/2016/10/20/test-stream.html b/content/blog/2016/10/20/test-stream.html index 58fc9d4..abc4563 100644 --- a/content/blog/2016/10/20/test-stream.html +++ b/content/blog/2016/10/20/test-stream.html @@ -220,7 +220,7 @@ reactions to speculative and late panes and dropped data. Element Timings -Elements arrive either behind, with, or after the watermark, which categorizes +Elements arrive either before, with, or after the watermark, which categorizes them into the âearlyâ, âon-timeâ, and âlateâ divisions. âLateâ elements can be further subdivided into âunobservablyâ, âobservablyâ, and âdroppablyâ late, depending on the window to which they are assigned and the maximum allowed http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/de57d1ae/content/feed.xml -- diff --git a/content/feed.xml b/content/feed.xml index 1e64366..a91a76e 100644 --- a/content/feed.xml +++ b/content/feed.xml @@ -80,7 +80,7 @@ reactions to speculative and late panes and dropped data.Element Timings
-Elements arrive either behind, with, or after the watermark, which categorizes +
Elements arrive either before, with, or after the watermark, which categorizes them into the âearlyâ, âon-timeâ, and âlateâ divisions. âLateâ elements can be further subdivided into âunobservablyâ, âobservablyâ, and âdroppablyâ late, depending on the window to which they are assigned and the maximum allowed
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/159ff482 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/159ff482 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/159ff482 Branch: refs/heads/asf-site Commit: 159ff4821ebfb63a93899b41000bb8d00fcfb978 Parents: fe87fb8 Author: Davor Bonaci Authored: Mon Nov 14 16:35:14 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 14 16:35:14 2016 -0800 -- content/documentation/runners/direct/index.html | 43 +++- 1 file changed, 41 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/159ff482/content/documentation/runners/direct/index.html -- diff --git a/content/documentation/runners/direct/index.html b/content/documentation/runners/direct/index.html index d47b51b..6cc31f9 100644 --- a/content/documentation/runners/direct/index.html +++ b/content/documentation/runners/direct/index.html @@ -6,7 +6,7 @@ - Apache Direct Runner + Direct Runner @@ -142,7 +142,46 @@ Using the Direct Runner -This page is under construction (https://issues.apache.org/jira/browse/BEAM-505";>BEAM-505). +The Direct Runner executes pipelines on your machine and is designed to validate that pipelines adhere to the Apache Beam model as closely as possible. Instead of focusing on efficient pipeline execution, the Direct Runner performs additional checks to ensure that users do not rely on semantics that are not guaranteed by the model. Some of these checks include: + + + enforcing immutability of elements + enforcing encodability of elements + elements are processed in an arbitrary order at all points + serialization of user functions (DoFn, CombineFn, etc.) + + +Using the Direct Runner for testing and development helps ensure that pipelines are robust across different Beam runners. In addition, debugging failed runs can be a non-trivial task when a pipeline executes on a remote cluster. Instead, it is often faster and simpler to perform local unit testing on your pipeline code. Unit testing your pipeline locally also allows you to use your preferred local debugging tools. + +Here are some resources with information about how to test your pipelines. + + Testing Unbounded Pipelines in Apache Beam talks about the use of Java classes PAssert and TestStream to test your pipelines. + The Apache Beam WordCount Example contains an example of logging and testing a pipeline with PAssert. + + +Direct Runner prerequisites and setup + +You must specify your dependency on the Direct Runner. + ++ + + + +Pipeline options for the Direct Runner + +When executing your pipeline from the command-line, set runner to direct. The default values for the other pipeline options are generally sufficient. + +See the reference documentation for the DirectOptionshttps://github.com/apache/incubator-beam/blob/python-sdk/sdks/python/apache_beam/utils/options.py";>PipelineOptions interface (and its subinterfaces) for defaults and the complete list of pipeline configuration options. + +Additional information and caveats + +Local execution is limited by the memory available in your local environment. It is highly recommended that you run your pipeline with data sets small enough to fit in local memory. You can create a small in-memory data set using a Createhttps://github.com/apache/incubator-beam/blob/python-sdk/sdks/python/apache_beam/transforms/core.py";>Create transform, or you can use a Readhttps://github.com/apache/incubator-beam/blob/python-sdk/sdks/python/apache_beam/io/iobase.py";>Read transform to work with small local or remote files. +org.apache.beam +beam-runners-direct-java +0.3.0-incubating +runtime +
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/832d2abe Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/832d2abe Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/832d2abe Branch: refs/heads/asf-site Commit: 832d2abe2aa43c5c0a14366dae574f01f57f4f0d Parents: 5fbc7b7 Author: Davor Bonaci Authored: Mon Nov 14 17:10:54 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 14 17:10:54 2016 -0800 -- .../documentation/runners/dataflow/index.html | 128 ++- 1 file changed, 126 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/832d2abe/content/documentation/runners/dataflow/index.html -- diff --git a/content/documentation/runners/dataflow/index.html b/content/documentation/runners/dataflow/index.html index aa403ec..507be0b 100644 --- a/content/documentation/runners/dataflow/index.html +++ b/content/documentation/runners/dataflow/index.html @@ -140,9 +140,133 @@ -Using the Cloud Dataflow Runner +Using the Google Cloud Dataflow Runner -This page is under construction (https://issues.apache.org/jira/browse/BEAM-508";>BEAM-508). +The Google Cloud Dataflow Runner uses the https://cloud.google.com/dataflow/service/dataflow-service-desc";>Cloud Dataflow managed service. When you run your pipeline with the Cloud Dataflow service, the runner uploads your executable code and dependencies to a Google Cloud Storage bucket and creates a Cloud Dataflow job, which executes your pipeline on managed resources in Google Cloud Platform. + +The Cloud Dataflow Runner and service are suitable for large scale, continuous jobs, and provide: + + + a fully managed service + https://cloud.google.com/dataflow/service/dataflow-service-desc#autoscaling";>autoscaling of the number of workers throughout the lifetime of the job + https://cloud.google.com/blog/big-data/2016/05/no-shard-left-behind-dynamic-work-rebalancing-in-google-cloud-dataflow";>dynamic work rebalancing + + +The Beam Capability Matrix documents the supported capabilities of the Cloud Dataflow Runner. + +Cloud Dataflow Runner prerequisites and setup +To use the Cloud Dataflow Runner, you must complete the following setup: + + + +Select or create a Google Cloud Platform Console project. + + +Enable billing for your project. + + +Enable required Google Cloud APIs: Cloud Dataflow, Compute Engine, Stackdriver Logging, Cloud Storage, and Cloud Storage JSON. You may need to enable additional APIs (such as BigQuery, Cloud Pub/Sub, or Cloud Datastore) if you use them in your pipeline code. + + +Install the Google Cloud SDK. + + +Create a Cloud Storage bucket. + + In the Google Cloud Platform Console, go to the Cloud Storage browser. + Click Create bucket. + In the Create bucket dialog, specify the following attributes: + + Name: A unique bucket name. Do not include sensitive information in the bucket name, as the bucket namespace is global and publicly visible. + Storage class: Multi-Regional + Location: Choose your desired location + + + Click Create. + + + + +For more information, see the Before you begin section of the https://cloud.google.com/dataflow/docs/quickstarts";>Cloud Dataflow quickstarts. + +Specify your dependency + +You must specify your dependency on the Cloud Dataflow Runner. + ++ + + + +Authentication + +Before running your pipeline, you must authenticate with the Google Cloud Platform. Run the following command to get https://developers.google.com/identity/protocols/application-default-credentials";>Application Default Credentials. + +gcloud auth application-default login + + + +Pipeline options for the Cloud Dataflow Runner + +When executing your pipeline with the Cloud Dataflow Runner, set these pipeline options. + + + + Field + Description + Default Value + + + runner + The pipeline runner to use. This option allows you to determine the pipeline runner at runtime. + Set to dataflow to run on the Cloud Dataflow Service. + + + project + The project ID for your Google Cloud Project. + If not set, defaults to the default project in the current environment. The default project is set via gcloud. + + + streaming + Whether streaming mode is enabled or disabled; true if enabled. Set to true if running pipelines with unbounded PCollections. + false + + + torg.apache.beam +beam-runners-google-cloud-dataflow-java +0.3.0-incubating +runtime +
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/d8cdb81a Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/d8cdb81a Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/d8cdb81a Branch: refs/heads/asf-site Commit: d8cdb81af56f2b5ba3b8411aa6951ca2688c880f Parents: 748fe82 Author: Davor Bonaci Authored: Mon Nov 14 18:23:16 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 14 18:23:16 2016 -0800 -- content/v2/css/beam.css | 6 +-- content/v2/images/beam-logo.png | Bin 3382 -> 0 bytes content/v2/index.html | 78 --- 3 files changed, 29 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/d8cdb81a/content/v2/css/beam.css -- diff --git a/content/v2/css/beam.css b/content/v2/css/beam.css index 7d318e3..11ef08f 100644 --- a/content/v2/css/beam.css +++ b/content/v2/css/beam.css @@ -245,9 +245,9 @@ code{font-size:14px}body{background-color:#f1f1f1} .header--navbar .navbar-inner .navbar-link.nav-search.js-highlight .navbar-search--icon{display:none} .header--navbar .navbar-inner .navbar-link.nav-search.js-highlight .search-input-close{display:inline-block;color:#eee;padding:4px 24px 3px;cursor:pointer}.header--navbar .search-dropdown--container{position:absolute;z-index:998;margin-top:-90px;background-color:#fff;width:100%;border-bottom:1px solid #ddd;transition:margin .25s;-webkit-transition:margin .25s;-moz-transition:margin .25s;-o-transition:margin .25s;-ms-transition:margin .25s}.header--navbar .search-dropdown--container.no-animation{transition:none;-webkit-transition:none;-moz-transition:none;-o-transition:none;-ms-transition:none}.header--navbar .search-dropdown--container .form-search{margin:0}.header--navbar .search-dropdown--container .form-search .search-form--form{background:rgba(0,0,0,0);border:0;box-shadow:none;color:#34302D;font-size:21px;line-height:29px;height:30px;padding:25px 0;width:93%}.header--navbar .search-dropdown--container .form-search .search-form--form::-webkit-input-placeholder{font-style:italic}. header--navbar .search-dropdown--container .form-search .search-form--form:-moz-placeholder{font-style:italic}.header--navbar .search-dropdown--container .form-search .search-form--form::-moz-placeholder{font-style:italic}.header--navbar .search-dropdown--container .form-search .search-form--form:-ms-input-placeholder{font-style:italic}.header--navbar .search-dropdown--container .form-search .search-form--submit{background-color:transparent;border:0;float:right;padding:28px 16px 24px 0}.header--navbar .search-dropdown--container .form-search .icon-search{font-size:22px;color:#34302d}.header--navbar .search-dropdown--container.js-show{margin-top:0} -a.beam-logo{background:url(../images/beam-logo.png) -1px -1px no-repeat} +a.beam-logo{background:url(../../images/logos/full-color/name-right/beam-logo-full-color-name-right.svg) -1px -1px no-repeat} -a.beam-logo span{display:block;width:160px;height:46px;background:url(../images/beam-logo.png) -1px -48px no-repeat;opacity:0;-moz-transition:opacity .12s ease-in-out;-webkit-transition:opacity .12s ease-in-out;-o-transition:opacity .12s ease-in-out} +a.beam-logo span{display:block;width:160px;height:46px;background:url(../../images/logos/full-color/name-right/beam-logo-full-color-name-right.svg) -1px -48px no-repeat;opacity:0;-moz-transition:opacity .12s ease-in-out;-webkit-transition:opacity .12s ease-in-out;-o-transition:opacity .12s ease-in-out} .right-pane-widget--container{border:1px solid #ccc;box-shadow:0 -4px 0 #34302d;margin:4px 0 40px}.right-pane-widget--container li{display:block;padding:12px 6.2%;line-height:14px;border-top:1px solid #fff}.right-pane-widget--container li [class^=icon-]{font-size:16px}.right-pane-widget--container li a{font-size:14px;line-height:14px;color:#5fa134}.right-pane-widget--container li:first-child{border-bottom:0}.right-pane-widget--container.secondary-nav li a{text-transform:uppercase;color:#888;font-family:Montserrat,sans-serif;font-weight:400;text-decoration:none}.right-pane-widget--container.secondary-nav li a:hover{color:#5fa134}.right-pane-widget--container.secondary-nav li.active a{font-weight:700;color:#34302d}.right-pane-widget--container.with-icon li{padding:12px 6.2% 12px 3%}.right-pane-widget--container.with-icon li a{margin-left:2%}.right-pane-widget--container.no-top-border{box-shadow:none}.header--content{background-color:#dedede;padding:60px 0}.header--content .header--con tent-subtitle--link{color:#8cc63f}.header--content .header--content-subtitle--link:hover{text-decoration:none;
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/bfbd64c3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/bfbd64c3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/bfbd64c3 Branch: refs/heads/asf-site Commit: bfbd64c327b3aea2bc02f5c1f829f6fc1229e703 Parents: a53e4d8 Author: Davor Bonaci Authored: Wed Nov 16 09:35:38 2016 -0800 Committer: Davor Bonaci Committed: Wed Nov 16 09:35:38 2016 -0800 -- content/get-started/quickstart/index.html | 42 -- 1 file changed, 39 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/bfbd64c3/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index 1ef229a..92176b5 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -222,7 +222,19 @@ MinimalWordCount.java WordCount.java -TODO BEAM-899 +$ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \ + -Dexec.args="--runner=FlinkRunner --inputFile=pom.xml --output=counts" -Pflink-runner + + + +$ mvn package -Pflink-runner +$ cp target/word-count-beam-bundled-0.1.jar /path/to/flink/lib/ +$ bin/flink run -c org.apache.beam.examples.WordCount lib/word-count-beam-0.1.jar \ +--inputFile=/path/to/quickstart/pom.xml \ +--output=/tmp/counts \ +--runner=org.apache.beam.runners.flink.FlinkRunner + +You can monitor the running job by visiting the Flink dashboard at http://:8081 @@ -248,7 +260,11 @@ MinimalWordCount.java WordCount.java -TODO BEAM-899 +$ ls counts* + + + +$ ls /tmp/counts* @@ -284,7 +300,27 @@ PAssert: 1 -TODO BEAM-899 +$ more counts* +The: 1 +api: 9 +old: 4 +Apache: 2 +limitations: 1 +bundled: 1 +Foundation: 1 +... + + + +$ more /tmp/counts* +The: 1 +api: 9 +old: 4 +Apache: 2 +limitations: 1 +bundled: 1 +Foundation: 1 +...
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/c219b493 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/c219b493 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/c219b493 Branch: refs/heads/asf-site Commit: c219b493269500f214b3a694f7f1559cc971245c Parents: 99ad787 Author: Davor Bonaci Authored: Wed Nov 16 09:43:00 2016 -0800 Committer: Davor Bonaci Committed: Wed Nov 16 09:43:00 2016 -0800 -- content/contribute/work-in-progress/index.html | 6 -- 1 file changed, 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/c219b493/content/contribute/work-in-progress/index.html -- diff --git a/content/contribute/work-in-progress/index.html b/content/contribute/work-in-progress/index.html index d88cf2e..5f6ee10 100644 --- a/content/contribute/work-in-progress/index.html +++ b/content/contribute/work-in-progress/index.html @@ -170,12 +170,6 @@ - Apache Apex Runner - https://github.com/apache/incubator-beam/tree/apex-runner";>apex-runner - https://issues.apache.org/jira/browse/BEAM/component/12331007";>runner-apex - - - - Apache Gearpump Runner https://github.com/apache/incubator-beam/tree/gearpump-runner";>gearpump-runner https://issues.apache.org/jira/browse/BEAM/component/12330829";>runner-gearpump
[2/3] incubator-beam-site git commit: Regenerate Website
Regenerate Website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/3f72e67a Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/3f72e67a Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/3f72e67a Branch: refs/heads/asf-site Commit: 3f72e67ac42e1fd094472c42bebe03520c8c98e4 Parents: b3cc5a2 Author: Aljoscha Krettek Authored: Mon Nov 21 10:35:15 2016 +0100 Committer: Aljoscha Krettek Committed: Mon Nov 21 10:35:15 2016 +0100 -- content/js/language-switch.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/3f72e67a/content/js/language-switch.js -- diff --git a/content/js/language-switch.js b/content/js/language-switch.js index 2c634fe..653cbcb 100644 --- a/content/js/language-switch.js +++ b/content/js/language-switch.js @@ -79,7 +79,6 @@ $(document).ready(function() { }, "toggle": function() { var pref=localStorage.getItem(this.dbKey) || this.default; - // Adjusting active elements in navigation header. $("." + this.wrapper + " li").removeClass("active").each(function() { if ($(this).data("type") === pref) { @@ -88,7 +87,7 @@ $(document).ready(function() { }); // Swapping visibility of code blocks. -$("[class^=" + prefix).hide(); +$(this.selector).hide(); $("." + pref).show(); }, "render": function(wrapper) { @@ -101,4 +100,4 @@ $(document).ready(function() { Switcher({"class-prefix":"language","default":"java"}).render(); Switcher({"class-prefix":"runner","default":"direct"}).render(); -}); \ No newline at end of file +});
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/b0f91e7f Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/b0f91e7f Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/b0f91e7f Branch: refs/heads/asf-site Commit: b0f91e7f11ec514d233f8bccb632676e37093eaa Parents: 7f7a8b2 Author: Davor Bonaci Authored: Mon Nov 21 17:03:28 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 21 17:03:28 2016 -0800 -- content/get-started/downloads/index.html | 33 +-- 1 file changed, 21 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/b0f91e7f/content/get-started/downloads/index.html -- diff --git a/content/get-started/downloads/index.html b/content/get-started/downloads/index.html index 5b89c96..0849509 100644 --- a/content/get-started/downloads/index.html +++ b/content/get-started/downloads/index.html @@ -167,28 +167,37 @@ Java with the DirectRunner, add the follo connectors or other extensions, and additional runners to execute your pipeline at scale. -Release Notes +API Stability -0.3.0-incubating -https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.3.0-incubating/apache-beam-0.3.0-incubating-source-release.zip&action=download";>Source code download +Apache Beam uses http://semver.org/";>semantic versioning. Version numbers use the form major.minor.incremental and are incremented as follows: - Release notes are available https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12338051";>in JIRA. + major version for incompatible API changes + minor version for new functionality added in a backward-compatible manner + incremental version for forward-compatible bug fixes +Please note that APIs marked @Experimental +may change at any point and are not guaranteed to remain compatible across versions. + +Additionally, any API may change before the first stable release, i.e., between versions denoted 0.x.y. + +Releases + +0.3.0-incubating +Official https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.3.0-incubating/apache-beam-0.3.0-incubating-source-release.zip&action=download";>source code download. + +https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12338051";>Release notes. + 0.2.0-incubating -https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.2.0-incubating/apache-beam-0.2.0-incubating-source-release.zip&action=download";>Source code download +Official https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.2.0-incubating/apache-beam-0.2.0-incubating-source-release.zip&action=download";>source code download. - - Release notes are available https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12335766";>in JIRA. - +https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12335766";>Release notes. 0.1.0-incubating -https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.1.0-incubating/apache-beam-0.1.0-incubating-source-release.zip&action=download";>Source code download +Official https://www.apache.org/dyn/closer.cgi?filename=incubator/beam/0.1.0-incubating/apache-beam-0.1.0-incubating-source-release.zip&action=download";>source code download. - - The first incubating release of Apache Beam. - +The first incubating release of Apache Beam.
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/66f2d553 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/66f2d553 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/66f2d553 Branch: refs/heads/asf-site Commit: 66f2d553bcc064d61a5ce1b63fa1919ade1048b2 Parents: 71ebc2d Author: Davor Bonaci Authored: Mon Nov 21 18:44:00 2016 -0800 Committer: Davor Bonaci Committed: Mon Nov 21 18:44:00 2016 -0800 -- content/get-started/quickstart/index.html | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/66f2d553/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index 92176b5..36dee90 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -168,14 +168,15 @@ The easiest way to get a copy of the WordCount pipeline is to use the following command to generate a simple Maven project that contains Beamâs WordCount examples and builds against the most recent Beam release: $ mvn archetype:generate \ + -DarchetypeRepository=https://repository.apache.org/content/groups/snapshots \ + -DarchetypeGroupId=org.apache.beam \ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \ -DarchetypeVersion=LATEST \ - -DarchetypeGroupId=org.apache.beam \ -DgroupId=org.example \ -DartifactId=word-count-beam \ -Dversion="0.1" \ - -DinteractiveMode=false \ - -Dpackage=org.apache.beam.examples + -Dpackage=org.apache.beam.examples \ + -DinteractiveMode=false
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/6a453509 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/6a453509 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/6a453509 Branch: refs/heads/asf-site Commit: 6a45350997258dbae8bf2ffa6b712a5f9bff7130 Parents: 5b3bda6 Author: Davor Bonaci Authored: Tue Nov 22 22:11:49 2016 -0800 Committer: Davor Bonaci Committed: Tue Nov 22 22:11:49 2016 -0800 -- .../pipelines/create-your-pipeline/index.html | 174 ++- .../pipelines/design-your-pipeline/index.html | 117 +++- .../pipelines/test-your-pipeline/index.html | 289 ++- content/images/design-your-pipeline-flatten.png | Bin 0 -> 47858 bytes content/images/design-your-pipeline-join.png| Bin 0 -> 41878 bytes content/images/design-your-pipeline-linear.png | Bin 0 -> 15218 bytes ...sign-your-pipeline-multiple-pcollections.png | Bin 0 -> 39095 bytes .../design-your-pipeline-side-outputs.png | Bin 0 -> 36451 bytes 8 files changed, 575 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6a453509/content/documentation/pipelines/create-your-pipeline/index.html -- diff --git a/content/documentation/pipelines/create-your-pipeline/index.html b/content/documentation/pipelines/create-your-pipeline/index.html index c3c2182..83e43f9 100644 --- a/content/documentation/pipelines/create-your-pipeline/index.html +++ b/content/documentation/pipelines/create-your-pipeline/index.html @@ -143,10 +143,182 @@ Create Your Pipeline + + Creating Your Pipeline Object + Configuring Pipeline Options + Setting PipelineOptions from Command-Line Arguments + Creating Custom Options + + + + + Reading Data Into Your Pipeline + Applying Transforms to Process Pipeline Data + Writing or Outputting Your Final Pipeline Data + Running Your Pipeline + Whatâs next + + +Your Beam program expresses a data processing pipeline, from start to finish. This section explains the mechanics of using the classes in the Beam SDKs to build a pipeline. To construct a pipeline using the classes in the Beam SDKs, your program will need to perform the following general steps: + + + Create a Pipeline object. + Use a Read or Create transform to create one or more PCollections for your pipeline data. + Apply transforms to each PCollection. Transforms can change, filter, group, analyze, or otherwise process the elements in a PCollection. Each transform creates a new output PCollection, to which you can apply additional transforms until processing is complete. + Write or otherwise output the final, transformed PCollections. + Run the pipeline. + + +Creating Your Pipeline Object + +A Beam program often starts by creating a Pipeline object. + +In the Beam SDKs, each pipeline is represented by an explicit object of type Pipeline. Each Pipeline object is an independent entity that encapsulates both the data the pipeline operates over and the transforms that get applied to that data. + +To create a pipeline, declare a Pipeline object, and pass it some configuration options, which are explained in a section below. You pass the configuration options by creating an object of type PipelineOptions, which you can build by using the static method PipelineOptionsFactory.create(). + +// Start by defining the options for the pipeline. +PipelineOptions options = PipelineOptionsFactory.create(); + +// Then create the pipeline. +Pipeline p = Pipeline.create(options); + + + +Configuring Pipeline Options + +Use the pipeline options to configure different aspects of your pipeline, such as the pipeline runner that will execute your pipeline and any runner-specific configuration required by the chosen runner. Your pipeline options will potentially include information such as your project ID or a location for storing files. + +When you run the pipeline on a runner of your choice, a copy of the PipelineOptions will be available to your code. For example, you can read PipelineOptions from a DoFnâs Context. + +Setting PipelineOptions from Command-Line Arguments + +While you can configure your pipeline by creating a PipelineOptions object and setting the fields directly, the Beam SDKs include a command-line parser that you can use to set fields in PipelineOptions using command-line arguments. + +To read options from the command-line, construct your PipelineOptions object as demonstrated in the following example code: + +MyOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().create(); + + + +This interprets co
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/ebf00421 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/ebf00421 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/ebf00421 Branch: refs/heads/asf-site Commit: ebf00421fe66c31d6c80e51df1d20bd702c20124 Parents: c79ea35 Author: Davor Bonaci Authored: Wed Nov 23 14:55:08 2016 -0800 Committer: Davor Bonaci Committed: Wed Nov 23 14:55:08 2016 -0800 -- .../capability/2016/03/17/capability-matrix.html| 5 - .../2016/04/03/presentation-materials.html | 5 - .../sdk/2016/02/25/python-sdk-now-public.html | 5 - content/beam/release/2016/06/15/first-release.html | 5 - .../2016/10/11/strata-hadoop-world-and-beam.html| 5 - .../update/website/2016/02/22/beam-has-a-logo.html | 5 - content/blog/2016/05/18/splitAtFraction-method.html | 5 - .../2016/05/27/where-is-my-pcollection-dot-map.html | 5 - .../2016/06/13/flink-batch-runner-milestone.html| 5 - content/blog/2016/08/03/six-months.html | 5 - content/blog/2016/10/20/test-stream.html| 5 - content/blog/index.html | 5 - content/coming-soon.html| 5 - content/contribute/contribution-guide/index.html| 5 - content/contribute/design-principles/index.html | 5 - content/contribute/index.html | 5 - content/contribute/logos/index.html | 5 - content/contribute/maturity-model/index.html| 5 - .../contribute/presentation-materials/index.html| 5 - content/contribute/release-guide/index.html | 5 - content/contribute/source-repository/index.html | 5 - content/contribute/team/index.html | 5 - content/contribute/testing/index.html | 5 - content/contribute/work-in-progress/index.html | 5 - content/documentation/index.html| 5 - .../pipelines/create-your-pipeline/index.html | 5 - .../pipelines/design-your-pipeline/index.html | 5 - .../pipelines/test-your-pipeline/index.html | 5 - content/documentation/programming-guide/index.html | 5 - content/documentation/resources/index.html | 5 - content/documentation/runners/apex/index.html | 5 - .../runners/capability-matrix/index.html| 5 - content/documentation/runners/dataflow/index.html | 5 - content/documentation/runners/direct/index.html | 5 - content/documentation/runners/flink/index.html | 5 - content/documentation/runners/spark/index.html | 5 - content/documentation/sdks/java/index.html | 5 - content/documentation/sdks/python/index.html| 5 - content/get-started/beam-overview/index.html| 5 - content/get-started/downloads/index.html| 5 - content/get-started/index.html | 5 - .../get-started/mobile-gaming-example/index.html| 5 - content/get-started/quickstart/index.html | 5 - content/get-started/support/index.html | 5 - content/get-started/wordcount-example/index.html| 9 ++--- content/images/external-link-icon.png | Bin 0 -> 1871 bytes content/index.html | 5 - content/privacy_policy/index.html | 5 - 48 files changed, 190 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/ebf00421/content/beam/capability/2016/03/17/capability-matrix.html -- diff --git a/content/beam/capability/2016/03/17/capability-matrix.html b/content/beam/capability/2016/03/17/capability-matrix.html index 191521f..841a5bc 100644 --- a/content/beam/capability/2016/03/17/capability-matrix.html +++ b/content/beam/capability/2016/03/17/capability-matrix.html @@ -78,7 +78,10 @@ SDKs Java SDK - Java SDK API Reference + Java SDK API Reference + Python SDK Runners http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/ebf00421/content/beam/capability/2016/04/03/presentation-materials.html -- diff --git a/content/beam/capability/2016/04/03/presentation-materials.html b/content/beam/capability/2016/04/
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/7105969e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/7105969e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/7105969e Branch: refs/heads/asf-site Commit: 7105969e7b4bb6bcb15b5febc8fab296f272a16b Parents: 236b3b6 Author: Davor Bonaci Authored: Wed Nov 23 17:11:19 2016 -0800 Committer: Davor Bonaci Committed: Wed Nov 23 17:11:19 2016 -0800 -- content/get-started/beam-overview/index.html | 12 ++- content/get-started/index.html | 30 content/index.html | 90 --- 3 files changed, 40 insertions(+), 92 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/7105969e/content/get-started/beam-overview/index.html -- diff --git a/content/get-started/beam-overview/index.html b/content/get-started/beam-overview/index.html index 928c5d1..ccb4980 100644 --- a/content/get-started/beam-overview/index.html +++ b/content/get-started/beam-overview/index.html @@ -146,7 +146,7 @@ Apache Beam Overview -Apache Beam is an open source, unified programming model that you can use to create a data processing pipeline. You start by building a program that defines the pipeline using one of the open source Beam SDKs. The pipeline is then executed by one of Beamâs supported distributed processing back-ends, which include http://flink.apache.org";>Apache Flink, http://spark.apache.org";>Apache Spark, and https://cloud.google.com/dataflow";>Google Cloud Dataflow. +Apache Beam is an open source, unified programming model that you can use to create a data processing pipeline. You start by building a program that defines the pipeline using one of the open source Beam SDKs. The pipeline is then executed by one of Beamâs supported distributed processing back-ends, which include http://apex.apache.org";>Apache Apex, http://flink.apache.org";>Apache Flink, http://spark.apache.org";>Apache Spark, and https://cloud.google.com/dataflow";>Google Cloud Dataflow. Beam is particularly useful for http://en.wikipedia.org/wiki/Embarassingly_parallel";>Embarrassingly Parallel data processing tasks, in which the problem can be decomposed into many smaller bundles of data that can be processed independently and in parallel. You can also use Beam for Extract, Transform, and Load (ETL) tasks and pure data integration. These tasks are useful for moving data between different storage media and data sources, transforming data into a more desirable format, or loading data onto a new system. @@ -187,7 +187,7 @@ Status - Google Cloud Dataflow + Apache Apex In Development @@ -198,10 +198,18 @@ Apache Spark In Development + + Google Cloud Dataflow + In Development + Note: You can always execute your pipeline locally for testing and debugging purposes. +Getting Started with Apache Beam + +Get started using Beam for your data processing tasks by following the Quickstart and the WordCount Examples Walkthrough. + http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/7105969e/content/get-started/index.html -- diff --git a/content/get-started/index.html b/content/get-started/index.html index c7d0626..8ab34c0 100644 --- a/content/get-started/index.html +++ b/content/get-started/index.html @@ -144,35 +144,33 @@ -Using Apache Beam +Get Started with Apache Beam Learn to use Beam to create data processing pipelines that run on supported processing back-ends: -Beam Overview +Beam Overview + Learn about the Beam model, the currently available Beam SDKs and Runners, and Beamâs native I/O connectors. -Quickstart -Learn how to set up a Beam project and run a simple example Beam pipeline on your local machine. +Quickstart -Release Information -Find information on the latest Beam releases, including versioning and release notes. +Learn how to set up a Beam project and run a simple example Beam pipeline on your local machine. Example Walkthroughs + See detailed walkthroughs of complete Beam pipelines. - WordCount: Simple example pipelines that demonstrate basic Beam programming, including debugging and testing - Mobile Gaming: A series of more advanced pipelines that demonstrate use cases in the mobile gaming domain + WordCount: Simple example pipelines that demonstrate basic Beam programming, including debugging and testing + Mobile Gaming: A series of more advanced pipelines that demonstrate use cases in the mobile gaming domain -Support -
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/24eb9127 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/24eb9127 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/24eb9127 Branch: refs/heads/asf-site Commit: 24eb91271785ff16d09c3b69959c7c6f8a9d7e20 Parents: 3627a44 Author: Davor Bonaci Authored: Wed Nov 23 22:07:33 2016 -0800 Committer: Davor Bonaci Committed: Wed Nov 23 22:07:33 2016 -0800 -- .../documentation/programming-guide/index.html | 501 ++- 1 file changed, 498 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/24eb9127/content/documentation/programming-guide/index.html -- diff --git a/content/documentation/programming-guide/index.html b/content/documentation/programming-guide/index.html index d941e42..aa97bb6 100644 --- a/content/documentation/programming-guide/index.html +++ b/content/documentation/programming-guide/index.html @@ -172,6 +172,7 @@ Using ParDo Using GroupByKey Using Combine + Using Flatten and Partition General Requirements for Writing User Code for Beam Transforms Side Inputs and Side Outputs @@ -364,7 +365,7 @@ -The resulting workflow graph from the branching pipeline abouve looks like this: +The resulting workflow graph from the branching pipeline above looks like this: [Branching Graph Graphic] @@ -382,7 +383,7 @@ ParDo GroupByKey Combine - Flatten + Flatten and Partition ParDo @@ -552,6 +553,270 @@ tree, [2] Using Combine +Combinehttps://github.com/apache/incubator-beam/blob/python-sdk/sdks/python/apache_beam/transforms/core.py";>Combine is a Beam transform for combining collections of elements or values in your data. Combine has variants that work on entire PCollections, and some that combine the values for each key in PCollections of key/value pairs. + +When you apply a Combine transform, you must provide the function that contains the logic for combining the elements or values. The combining function should be commutative and associative, as the function is not necessarily invoked exactly once on all values with a given key. Because the input data (including the value collection) may be distributed across multiple workers, the combining function might be called multiple times to perform partial combining on subsets of the value collection. The Beam SDK also provides some pre-built combine functions for common numeric combination operations such as sum, min, and max. + +Simple combine operations, such as sums, can usually be implemented as a simple function. More complex combination operations might require you to create a subclass of CombineFn that has an accumulation type distinct from the input/output type. + +Simple Combinations Using Simple Functions + +The following example code shows a simple combine function. + +// Sum a collection of Integer values. The function SumInts implements the interface SerializableFunction. +public static class SumInts implements SerializableFunction, Integer> { + @Override + public Integer apply(Iterable input) { +int sum = 0; +for (int item : input) { + sum += item; +} +return sum; + } +} + + + +# A bounded sum of positive integers. +def bounded_sum(values, bound=500): + return min(sum(values), bound) + + + +Advanced Combinations using CombineFn + +For more complex combine functions, you can define a subclass of CombineFn. You should use CombineFn if the combine function requires a more sophisticated accumulator, must perform additional pre- or post-processing, might change the output type, or takes the key into account. + +A general combining operation consists of four operations. When you create a subclass of CombineFn, you must provide four operations by overriding the corresponding methods: + + + +Create Accumulator creates a new âlocalâ accumulator. In the example case, taking a mean average, a local accumulator tracks the running sum of values (the numerator value for our final average division) and the number of values summed so far (the denominator value). It may be called any number of times in a distributed fashion. + + +Add Input adds an input element to an accumulator, returning the accumulator value. In our example, it would update the sum and increment the count. It may also be invoked in parallel. + + +Merge Accumulators merges several accumulators into a single accumulator; this is how data in multiple accumulators is combined before the final calculation. In the case of the mean average computation, t
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/9d43c74e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/9d43c74e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/9d43c74e Branch: refs/heads/asf-site Commit: 9d43c74e910fba9481e427a6bb33ea9db972d260 Parents: 5438941 Author: Davor Bonaci Authored: Fri Dec 2 10:32:52 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 2 10:32:52 2016 -0800 -- content/contribute/contribution-guide/index.html | 4 ++-- content/contribute/maturity-model/index.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/9d43c74e/content/contribute/contribution-guide/index.html -- diff --git a/content/contribute/contribution-guide/index.html b/content/contribute/contribution-guide/index.html index c27f9ca..87d5f74 100644 --- a/content/contribute/contribution-guide/index.html +++ b/content/contribute/contribution-guide/index.html @@ -542,7 +542,7 @@ $ git merge --no-ff \ -Always use --no-ff option and the specific commit message âThis closes #ââ â it ensures proper marking in the tooling. It would be nice to include additional information in the merge commit message, such as the title and summary of the pull request. +Always use --no-ff option and the specific commit message âThis closes # â â it ensures proper marking in the tooling. It would be nice to include additional information in the merge commit message, such as the title and summary of the pull request. At this point, you want to ensure everything is right. Test it with mvn verify. Run gitk or git log --graph, etc. When you are happy with how it looks, push it. This is the point of no return â proceed with caution. @@ -556,7 +556,7 @@ $ git merge --no-ff \ The project management committee (PMC) can grant more rights to a contributor, such as commit access or decision power, and recognize them as new committers or PMC members. -The PMC periodically discusses this topic and privately votes to grant more rights to a contributor. If the vote passess, the contributor is invited to accept or reject the nomination. Once accepted, the PMC announces the decision publicly and updates the list of team member accordingly. +The PMC periodically discusses this topic and privately votes to grant more rights to a contributor. If the vote passes, the contributor is invited to accept or reject the nomination. Once accepted, the PMC announces the decision publicly and updates the list of team member accordingly. The key to the selection process is http://apache.org/foundation/how-it-works.html#meritocracy";>Meritocracy, literally government by merit. Contributors earn merit in many ways: contributing code, testing releases, participating in documentation effort, answering user questions, debating design proposals, triaging issues, evangelizing the project, growing user base, and any other action that benefits the project as a whole. http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/9d43c74e/content/contribute/maturity-model/index.html -- diff --git a/content/contribute/maturity-model/index.html b/content/contribute/maturity-model/index.html index 0f95fec..76dbc0c 100644 --- a/content/contribute/maturity-model/index.html +++ b/content/contribute/maturity-model/index.html @@ -286,7 +286,7 @@ QU50 The project strives to respond to documented bug reports in a timely manner. - YES. The project has resolved https://issues.apache.org/jira/browse/BEAM-984?jql%3Dproject%2520%253D%2520BEAM%2520AND%2520status%2520in%2520(Resolved%252C%2520Closed)">509 issues during incubation.Even further, https://issues.apache.org/jira/browse/BEAM/?selectedTab%3Dcom.atlassian.jira.jira-projects-plugin:components-panel=undefined&selectedTab=com.atlassian.jira.jira-projects-plugin:components-panel";>all project components have designated a single committer who gets assigned all newly filed issues for a triage/re-assignment to ensure timely action. + YES. The project has resolved https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Resolved%2C%20Closed)">550 issues during incubation.Even further, https://issues.apache.org/jira/browse/BEAM/?selectedTab%3Dcom.atlassian.jira.jira-projects-plugin:components-panel=undefined&selectedTab=com.atlassian.jira.jira-projects-plugin:components-panel";>all project components have designated a single committer who gets assigned all newly filed issue
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/18d5db7e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/18d5db7e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/18d5db7e Branch: refs/heads/asf-site Commit: 18d5db7e2e344e4a1da3405a91e675b1f6cd13a2 Parents: f92e07c Author: Davor Bonaci Authored: Fri Dec 2 13:44:07 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 2 13:44:07 2016 -0800 -- content/get-started/quickstart/index.html | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/18d5db7e/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index 218863a..b1b33ab 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -232,12 +232,9 @@ MinimalWordCount.java WordCount.java -$ mvn package -Pflink-runner -$ cp target/word-count-beam-bundled-0.1.jar /path/to/flink/lib/ -$ bin/flink run -c org.apache.beam.examples.WordCount lib/word-count-beam-0.1.jar \ ---inputFile=/path/to/quickstart/pom.xml \ ---output=/tmp/counts \ ---runner=org.apache.beam.runners.flink.FlinkRunner +$ mvn package exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \ + -Dexec.args="--runner=FlinkRunner --flinkMaster=--filesToStage=target/word-count-beam-bundled-0.1.jar \ + --inputFile=/path/to/quickstart/pom.xml --output=/tmp/counts" -Pflink-runner You can monitor the running job by visiting the Flink dashboard at http:// :8081
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/05f74b91 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/05f74b91 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/05f74b91 Branch: refs/heads/asf-site Commit: 05f74b911e10fc861b6394ae323175e317f8b380 Parents: 759875c Author: Davor Bonaci Authored: Fri Dec 9 11:27:27 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 9 11:27:27 2016 -0800 -- content/get-started/quickstart/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/05f74b91/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index b1b33ab..36bf203 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -218,7 +218,7 @@ MinimalWordCount.java WordCount.java $ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \ - -Dexec.args="--inputFile=pom.xml --output=counts" + -Dexec.args="--inputFile=pom.xml --output=counts" -Pdirect-runner
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/ad33f2a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/ad33f2a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/ad33f2a8 Branch: refs/heads/asf-site Commit: ad33f2a8d5d8c11dc6ce04f61ed9a93a7e2c469a Parents: 5047903 Author: Davor Bonaci Authored: Fri Dec 9 12:06:06 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 9 12:06:06 2016 -0800 -- content/documentation/resources/index.html| 128 +++-- content/images/resources/se-radio-podcast.png | Bin 0 -> 165792 bytes content/images/resources/streaming-101.png| Bin 0 -> 127023 bytes content/images/resources/streaming-102.png| Bin 0 -> 126182 bytes 4 files changed, 120 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/ad33f2a8/content/documentation/resources/index.html -- diff --git a/content/documentation/resources/index.html b/content/documentation/resources/index.html index b88d003..128f5cc 100644 --- a/content/documentation/resources/index.html +++ b/content/documentation/resources/index.html @@ -146,17 +146,129 @@ Additional Resources for Learning about Apache Beam -This page is under construction (https://issues.apache.org/jira/browse/BEAM-509";>BEAM-509). +This page provides links to some of our favorite videos, podcasts, and articles that will help you get started and learn more about Apache Beam. -Hereâs some links to some of our favorite articles and videos get you started. - - - https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101";>The world beyond batch: Streaming 101 - https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-102";>The world beyong batch: Streaming 102 - https://cloud.google.com/dataflow/blog/dataflow-beam-and-spark-comparison";>Dataflow/Beam & Spark: A Programming Model Comparison - http://googlecloudplatform.blogspot.com/2016/01/Dataflow-and-open-source-proposal-to-join-the-Apache-Incubator.html";>Dataflow and open source - proposal to join the Apache Incubator + + General + Fundamentals of Stream Processing with Apache Beam + Apache Beam: A Unified Model for Batch and Streaming Data Processing + Fundamentals of Stream Processing with Apache Beam + The World Beyond Batch: Streaming 101 + The World Beyond Batch: Streaming 102 + Software Engineering Radio Podcast Episode 272: Apache Beam + + + Beam & Friends + Flink and Beam: Current State & Roadmap + Lessons learned from developing a stream processing platform at scale + + + Technical Details + Watermarks: Time and Progress in Apache Beam and Beyond + Triggers in Apache Beam + + +General + +The following resources provide general overviews and fundamentals of Apache Beam. + +Fundamentals of Stream Processing with Apache Beam + +Data Science Summit, Jerusalem, 2016 + +Presented by Tyler Akidau, Apache Beam PPMC member + +https://www.youtube.com/embed/V35MwYcXEX0"; frameborder="0" allowfullscreen=""> + + +Apache Beam: A Unified Model for Batch and Streaming Data Processing + +Hadoop Summit, San Jose, CA, 2016 + +Presented by Davor Bonacci, Apache Beam PPMC member + +https://www.youtube.com/embed/7DZ8ONmeP5A"; frameborder="0" allowfullscreen=""> + + +Fundamentals of Stream Processing with Apache Beam + +@Scale Conference, San Jose, CA, 2016 + +Presented by Dan Halperin, Apache Beam PPMC member + +https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fatscaleevents%2Fvideos%2F1775945569345206%2F&show_text=0&width=560"; width="560" height="315" style="border:none;overflow:hidden" scrolling="no" frameborder="0" allowtransparency="true" allowfullscreen="true"> + + +The World Beyond Batch: Streaming 101 + +Written by Tyler Akidau, Apache Beam PPMC member + +Click on the image to read the article. + +https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101"; target="_blank"> + + +The World Beyond Batch: Streaming 102 + +Written by Tyler Akidau, Apache Beam PPMC member + +Click on the image to read the article. + +https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-102"; target="_blank"> + + +Software Engineering Radio Podcast Episode 272: Apache Beam + +Presented by Frances Perry, Apache Beam PPMC member + +http://www.se-radio.net/2016/10/se-radio-episode-272-frances-perry-on-apache-beam/"; target="_blank"> + + +Beam & Friends + +The following resources present Apache Beam partnerships. + +Flink and Beam: Current State & Roadmap + +Flink Forward, Berlin, 2016 + +Presented by Maximilian
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/6b76c3f4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/6b76c3f4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/6b76c3f4 Branch: refs/heads/asf-site Commit: 6b76c3f4e0b15236d433909732453e95b635ce87 Parents: 7a9190c Author: Davor Bonaci Authored: Fri Dec 9 12:49:30 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 9 12:49:30 2016 -0800 -- content/documentation/runners/spark/index.html | 157 +++- 1 file changed, 156 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6b76c3f4/content/documentation/runners/spark/index.html -- diff --git a/content/documentation/runners/spark/index.html b/content/documentation/runners/spark/index.html index 521cf5d..23960e0 100644 --- a/content/documentation/runners/spark/index.html +++ b/content/documentation/runners/spark/index.html @@ -146,7 +146,162 @@ Using the Apache Spark Runner -This page is under construction (https://issues.apache.org/jira/browse/BEAM-507";>BEAM-507). +The Apache Spark Runner can be used to execute Beam pipelines using http://spark.apache.org/";>Apache Spark. +The Spark Runner can execute Spark pipelines just like a native Spark application; deploying a self-contained application for local mode, running on Sparkâs Standalone RM, or using YARN or Mesos. + +The Spark Runner executes Beam pipelines on top of Apache Spark, providing: + + + Batch and streaming (and combined) pipelines. + The same fault-tolerance http://spark.apache.org/docs/1.6.3/streaming-programming-guide.html#fault-tolerance-semantics";>guarantees as provided by RDDs and DStreams. + The same http://spark.apache.org/docs/1.6.3/security.html";>security features Spark provides. + Built-in metrics reporting using Sparkâs metrics system, which reports Beam Aggregators as well. + Native support for Beam side-inputs via sparkâs Broadcast variables. + + +The Beam Capability Matrix documents the currently supported capabilities of the Spark Runner. + +Note: support for the Beam Model in streaming is currently experimental, follow-up in the mailing list for status updates. + +Spark Runner prerequisites and setup + +The Spark runner currently supports Sparkâs 1.6 branch, and more specifically any version greater than 1.6.0. + +You can add a dependency on the latest version of the Spark runner by adding to your pom.xml the following: ++ + + + +Deploying Spark with your application + +In some cases, such as running in local mode/Standalone, your (self-contained) application would be required to pack Spark by explicitly adding the following dependencies in your pom.xml: +org.apache.beam +beam-runners-spark +0.3.0-incubating ++ + +org.apache.spark +spark-core_2.10 +${spark.version} ++ + + + +And shading the application jar using the maven shade plugin: +org.apache.spark +spark-streaming_2.10 +${spark.version} ++ + + + +After running mvn package, run ls target and you should see (assuming your artifactId is beam-examples and the version is 1.0.0): +beam-examples-1.0.0-shaded.jar + + + +To run against a Standalone cluster simply run: +spark-submit --class com.beam.examples.BeamPipeline --master spark://HOST:PORT target/beam-examples-1.0.0-shaded.jar --runner=SparkRunner + + + +Running on a pre-deployed Spark cluster + +Deploying your Beam pipeline on a cluster that already has a Spark deployment (Spark classes are available in container classpath) does not require any additional deporg.apache.maven.plugins +maven-shade-plugin ++ +false ++ ++ +*:* ++ +META-INF/*.SF +META-INF/*.DSA +META-INF/*.RSA ++ ++ +package ++ +shade ++ +true +shaded +
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/c5f8a46e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/c5f8a46e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/c5f8a46e Branch: refs/heads/asf-site Commit: c5f8a46e8298bd23565552ed4c1e5deed2e82d25 Parents: b68b680 Author: Davor Bonaci Authored: Fri Dec 9 14:44:42 2016 -0800 Committer: Davor Bonaci Committed: Fri Dec 9 14:44:42 2016 -0800 -- content/get-started/mobile-gaming-example/index.html | 8 content/get-started/wordcount-example/index.html | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/c5f8a46e/content/get-started/mobile-gaming-example/index.html -- diff --git a/content/get-started/mobile-gaming-example/index.html b/content/get-started/mobile-gaming-example/index.html index ebd4b9f..ada7ce7 100644 --- a/content/get-started/mobile-gaming-example/index.html +++ b/content/get-started/mobile-gaming-example/index.html @@ -255,7 +255,7 @@ } @Override - public PCollection> apply( + public PCollection > expand( PCollection gameInfo) { return gameInfo @@ -518,7 +518,7 @@ } @Override - public PCollection > apply(PCollection input) { + public PCollection > expand(PCollection input) { return input.apply("LeaderboardUserGlobalWindow", Window. into(new GlobalWindows()) // Get periodic results every ten minutes. @@ -570,7 +570,7 @@ } @Override - public PCollection > apply(PCollection infos) { + public PCollection > expand(PCollection infos) { return infos.apply("LeaderboardTeamFixedWindows", Window. into(FixedWindows.of(teamWindowDuration)) // We will get early (speculative) results as well as cumulative @@ -628,7 +628,7 @@ private static final double SCORE_WEIGHT = 2.5; @Override - public PCollection > apply(PCollection > userScores) { + public PCollection > expand(PCollection > userScores) { // Get the sum of scores for each user. PCollection > sumScores = userScores http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/c5f8a46e/content/get-started/wordcount-example/index.html -- diff --git a/content/get-started/wordcount-example/index.html b/content/get-started/wordcount-example/index.html index 5394081..5132240 100644 --- a/content/get-started/wordcount-example/index.html +++ b/content/get-started/wordcount-example/index.html @@ -373,7 +373,7 @@ Figure 1: The pipeline data flow. public static class CountWords extends PTransform , PCollection >> { @Override - public PCollection > apply(PCollection lines) { + public PCollection > expand(PCollection lines) { // Convert lines of text into individual words. PCollection words = lines.apply(
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/2245a27d Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/2245a27d Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/2245a27d Branch: refs/heads/asf-site Commit: 2245a27d028c967ada0c8b22d0ebf20d2acb8d62 Parents: a5dacb9 Author: Davor Bonaci Authored: Sat Dec 10 13:20:58 2016 -0800 Committer: Davor Bonaci Committed: Sat Dec 10 13:20:58 2016 -0800 -- content/get-started/quickstart/index.html | 18 ++ 1 file changed, 14 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/2245a27d/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index 36bf203..cc5e1ed 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -186,7 +186,7 @@ This will create a directory word-count-beam that contains a simple pom.xml and a series of example pipelines that count words in text files. -$ cd beam-word-count/ +$ cd word-count-beam/ $ ls pom.xmlsrc @@ -240,7 +240,8 @@ You can monitor the running job by visiting the Flink dashboard at http://
[2/3] incubator-beam-site git commit: Regenerate website
Regenerate website Project: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/commit/d1b19ee3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/tree/d1b19ee3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam-site/diff/d1b19ee3 Branch: refs/heads/asf-site Commit: d1b19ee3997c5fb1518074d715f74153c9f0d2f7 Parents: 5eaa92b Author: Davor Bonaci Authored: Mon Dec 12 17:33:41 2016 -0800 Committer: Davor Bonaci Committed: Mon Dec 12 17:33:41 2016 -0800 -- content/get-started/quickstart/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/d1b19ee3/content/get-started/quickstart/index.html -- diff --git a/content/get-started/quickstart/index.html b/content/get-started/quickstart/index.html index cc5e1ed..2e2fc6a 100644 --- a/content/get-started/quickstart/index.html +++ b/content/get-started/quickstart/index.html @@ -247,7 +247,8 @@ You can monitor the running job by visiting the Flink dashboard at http:///tmp \ - --inputFile=gs://apache-beam-samples/shakespeare/* --output=gs:// /counts" + --inputFile=gs://apache-beam-samples/shakespeare/* --output=gs:// /counts" \ + -Pdataflow-runner