Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Flink #3954

[Apache Jenkins Server]

See 

Build failed in Jenkins: beam_PostCommit_Java_ValidatesRunner_Apex #2494

[Apache Jenkins Server]

See 


--
[...truncated 43.18 KB...]
2017-09-29T06:02:30.532 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/maven/enforcer/enforcer-api/3.0.0-M1/enforcer-api-3.0.0-M1.pom
 (3 KB at 97.9 KB/sec)
2017-09-29T06:02:30.539 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-utils/3.0.24/plexus-utils-3.0.24.pom
2017-09-29T06:02:30.569 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-utils/3.0.24/plexus-utils-3.0.24.pom
 (5 KB at 134.3 KB/sec)
2017-09-29T06:02:30.571 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/4.0/plexus-4.0.pom
2017-09-29T06:02:30.603 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/4.0/plexus-4.0.pom
 (21 KB at 656.1 KB/sec)
2017-09-29T06:02:30.608 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-component-annotations/1.6/plexus-component-annotations-1.6.pom
2017-09-29T06:02:30.635 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-component-annotations/1.6/plexus-component-annotations-1.6.pom
 (748 B at 27.1 KB/sec)
2017-09-29T06:02:30.638 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-containers/1.6/plexus-containers-1.6.pom
2017-09-29T06:02:30.665 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-containers/1.6/plexus-containers-1.6.pom
 (4 KB at 136.2 KB/sec)
2017-09-29T06:02:30.668 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/3.3.2/plexus-3.3.2.pom
2017-09-29T06:02:30.702 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/3.3.2/plexus-3.3.2.pom
 (22 KB at 618.4 KB/sec)
2017-09-29T06:02:30.707 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.pom
2017-09-29T06:02:30.735 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.pom
 (2 KB at 43.1 KB/sec)
2017-09-29T06:02:30.737 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-containers/1.0.3/plexus-containers-1.0.3.pom
2017-09-29T06:02:30.765 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-containers/1.0.3/plexus-containers-1.0.3.pom
 (492 B at 17.2 KB/sec)
2017-09-29T06:02:30.767 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/1.0.4/plexus-1.0.4.pom
2017-09-29T06:02:30.794 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus/1.0.4/plexus-1.0.4.pom
 (6 KB at 207.5 KB/sec)
2017-09-29T06:02:30.797 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/junit/junit/4.11/junit-4.11.pom
2017-09-29T06:02:30.825 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/junit/junit/4.11/junit-4.11.pom (3 KB at 
81.8 KB/sec)
2017-09-29T06:02:30.828 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.pom
2017-09-29T06:02:30.857 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.pom
 (766 B at 25.8 KB/sec)
2017-09-29T06:02:30.859 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-parent/1.3/hamcrest-parent-1.3.pom
2017-09-29T06:02:30.885 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-parent/1.3/hamcrest-parent-1.3.pom
 (2 KB at 74.1 KB/sec)
2017-09-29T06:02:30.889 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/classworlds/classworlds/1.1-alpha-2/classworlds-1.1-alpha-2.pom
2017-09-29T06:02:30.916 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/classworlds/classworlds/1.1-alpha-2/classworlds-1.1-alpha-2.pom
 (4 KB at 113.1 KB/sec)
2017-09-29T06:02:30.919 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven-project/2.0.9/maven-project-2.0.9.pom
2017-09-29T06:02:30.948 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven-project/2.0.9/maven-project-2.0.9.pom
 (3 KB at 91.3 KB/sec)
2017-09-29T06:02:30.952 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven/2.0.9/maven-2.0.9.pom
2017-09-29T06:02:30.981 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven/2.0.9/maven-2.0.9.pom
 (19 KB at 636.7 KB/sec)
2017-09-29T06:02:30.986 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/8/maven-parent-8.pom
2017-09-29T06:02:31.020 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/8/maven-parent-8.pom
 (24 KB at 785.6 KB/sec)
2017-09-29T06:02:31.024 [INFO] Downloading

[GitHub] beam pull request #3920: Package util

[kennknowles]

GitHub user kennknowles opened a pull request:

https://github.com/apache/beam/pull/3920

Package util

Follow this checklist to help us incorporate your contribution quickly and 
easily:

 - [ ] Make sure there is a [JIRA 
issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the 
change (usually before you start working on it).  Trivial changes like typos do 
not require a JIRA issue.  Your pull request should address just this issue, 
without pulling in other changes.
 - [ ] Each commit in the pull request should have a meaningful subject 
line and body.
 - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in 
ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA 
issue.
 - [ ] Write a pull request description that is detailed enough to 
understand what the pull request does, how, and why.
 - [ ] Run `mvn clean verify` to make sure basic checks pass. A more 
thorough check will be performed on your pull request automatically.
 - [ ] If this contribution is large, please file an Apache [Individual 
Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/beam PackageUtil

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3920.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3920


commit f32a34ec47b6b49d3818c65647ff5af5c92e2549
Author: Kenneth Knowles 
Date:   2017-09-29T02:35:20Z

Make PackageUtil a proper class encapsulating its ExecutorService

commit bfd5fb9dcb35c8624ef85cb9b01b1d56e535c126
Author: Kenneth Knowles 
Date:   2017-09-29T03:03:51Z

Use AutoValue for Dataflow PackageAttributes

commit 06c8f8ef8381086e1f7d1333c648f0408ab12bf0
Author: Kenneth Knowles 
Date:   2017-09-29T03:07:16Z

Refactor PackageUtil for more and simpler asynchrony




---

[GitHub] beam pull request #3919: Test PR for some python fixes

[jasonkuster]

GitHub user jasonkuster opened a pull request:

https://github.com/apache/beam/pull/3919

Test PR for some python fixes

Follow this checklist to help us incorporate your contribution quickly and 
easily:

 - [ ] Make sure there is a [JIRA 
issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the 
change (usually before you start working on it).  Trivial changes like typos do 
not require a JIRA issue.  Your pull request should address just this issue, 
without pulling in other changes.
 - [ ] Each commit in the pull request should have a meaningful subject 
line and body.
 - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in 
ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA 
issue.
 - [ ] Write a pull request description that is detailed enough to 
understand what the pull request does, how, and why.
 - [ ] Run `mvn clean verify` to make sure basic checks pass. A more 
thorough check will be performed on your pull request automatically.
 - [ ] If this contribution is large, please file an Apache [Individual 
Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/jasonkuster/beam python-fix

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3919.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3919


commit 834517157264f90abd6466a041b6dd912cb10ad1
Author: Jason Kuster 
Date:   2017-09-28T22:34:23Z

Allow users to skip Java or Python branch via a comment. Also use try/catch 
blocks to stop non-blocking branches from stopping the build.

Signed-off-by: Jason Kuster 

commit 5cdae8872724b411e3e5b2e7c1e723279eac117a
Author: Jason Kuster 
Date:   2017-09-28T22:55:11Z

Modify trigger phrase to require specifying which language.

Signed-off-by: Jason Kuster 

commit 9625de53ee1806f9dce8a896c14132066ee51d41
Author: Jason Kuster 
Date:   2017-09-29T01:17:07Z

Introduce a more sustainable way to add new SDKs.

Signed-off-by: Jason Kuster 

commit daceeea5cd32a9f246a3b95961aa89002725f17c
Author: Jason Kuster 
Date:   2017-09-29T02:25:53Z

test changing some python things to make build faster

Signed-off-by: Jason Kuster 




---

[beam-site] 01/01: Prepare repository for deployment.

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit ec964faf8a02e1e1ba459cc1722d59705cda766b
Author: Mergebot 
AuthorDate: Fri Sep 29 01:39:48 2017 +

Prepare repository for deployment.
---
 content/beam/capability/2016/03/17/capability-matrix.html  | 1 +
 content/beam/capability/2016/04/03/presentation-materials.html | 1 +
 content/beam/python/sdk/2016/02/25/python-sdk-now-public.html  | 1 +
 content/beam/release/2016/06/15/first-release.html | 1 +
 content/beam/update/2016/10/11/strata-hadoop-world-and-beam.html   | 1 +
 content/beam/update/website/2016/02/22/beam-has-a-logo.html| 1 +
 content/blog/2016/05/18/splitAtFraction-method.html| 1 +
 content/blog/2016/05/27/where-is-my-pcollection-dot-map.html   | 1 +
 content/blog/2016/06/13/flink-batch-runner-milestone.html  | 1 +
 content/blog/2016/08/03/six-months.html| 1 +
 content/blog/2016/10/20/test-stream.html   | 1 +
 content/blog/2017/01/09/added-apex-runner.html | 1 +
 content/blog/2017/01/10/beam-graduates.html| 1 +
 content/blog/2017/02/01/graduation-media-recap.html| 1 +
 content/blog/2017/02/13/stateful-processing.html   | 1 +
 content/blog/2017/03/16/python-sdk-release.html| 1 +
 content/blog/2017/05/17/beam-first-stable-release.html | 1 +
 content/blog/2017/08/16/splittable-do-fn.html  | 1 +
 content/blog/2017/08/28/timely-processing.html | 1 +
 content/blog/index.html| 1 +
 content/coming-soon.html   | 1 +
 content/contribute/contribution-guide/index.html   | 1 +
 content/contribute/design-principles/index.html| 1 +
 content/contribute/docker-images/index.html| 1 +
 content/contribute/index.html  | 1 +
 content/contribute/logos/index.html| 1 +
 content/contribute/maturity-model/index.html   | 1 +
 content/contribute/presentation-materials/index.html   | 1 +
 content/contribute/ptransform-style-guide/index.html   | 1 +
 content/contribute/release-guide/index.html| 1 +
 content/contribute/runner-guide/index.html | 1 +
 content/contribute/source-repository/index.html| 1 +
 content/contribute/team/index.html | 1 +
 content/contribute/testing/index.html  | 1 +
 content/contribute/work-in-progress/index.html | 1 +
 content/documentation/dsls/sql/index.html  | 1 +
 content/documentation/index.html   | 2 ++
 content/documentation/io/authoring-java/index.html | 1 +
 content/documentation/io/authoring-overview/index.html | 1 +
 content/documentation/io/authoring-python/index.html   | 1 +
 content/documentation/io/built-in/hadoop/index.html| 1 +
 content/documentation/io/built-in/index.html   | 1 +
 content/documentation/io/contributing/index.html   | 1 +
 content/documentation/io/io-toc/index.html | 1 +
 content/documentation/io/testing/index.html| 1 +
 content/documentation/pipelines/create-your-pipeline/index.html| 1 +
 content/documentation/pipelines/design-your-pipeline/index.html| 1 +
 content/documentation/pipelines/test-your-pipeline/index.html  | 1 +
 content/documentation/programming-guide/index.html | 1 +
 content/documentation/resources/index.html | 1 +
 content/documentation/runners/apex/index.html  | 1 +
 content/documentation/runners/capability-matrix/index.html | 1 +
 content/documentation/runners/dataflow/index.html  | 1 +
 content/documentation/runners/direct/index.html| 1 +
 content/documentation/runners/flink/index.html | 1 +
 content/documentation/runners/gearpump/index.html  | 1 +
 content/documentation/runners/spark/index.html | 1 +
 content/documentation/sdks/java-extensions/index.html  | 1 +
 content/documentation/sdks/java/index.html | 1 +
 content/documentation/sdks/python-custom-io/index.html | 1 +
 content/documentation/sdks/python-pipeline-dependencies/index.html | 1 +
 content/documentation/sdks/python-type-safety/index.html   | 1 +
 content/documentation/sdks/python/index.html   | 1 +

[beam-site] branch asf-site updated (4052af2 -> ec964fa)

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a change to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git.


from 4052af2  Prepare repository for deployment.
 add 0b495f8  Add new execution model page and associated images
 add fa4cf1e  Update with review feedback and new diagrams
 add 708bd58  More updates from review feedback
 add 7a1740b  This closes #326
 new ec964fa  Prepare repository for deployment.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../capability/2016/03/17/capability-matrix.html   |   1 +
 .../2016/04/03/presentation-materials.html |   1 +
 .../sdk/2016/02/25/python-sdk-now-public.html  |   1 +
 content/beam/release/2016/06/15/first-release.html |   1 +
 .../2016/10/11/strata-hadoop-world-and-beam.html   |   1 +
 .../update/website/2016/02/22/beam-has-a-logo.html |   1 +
 .../blog/2016/05/18/splitAtFraction-method.html|   1 +
 .../05/27/where-is-my-pcollection-dot-map.html |   1 +
 .../2016/06/13/flink-batch-runner-milestone.html   |   1 +
 content/blog/2016/08/03/six-months.html|   1 +
 content/blog/2016/10/20/test-stream.html   |   1 +
 content/blog/2017/01/09/added-apex-runner.html |   1 +
 content/blog/2017/01/10/beam-graduates.html|   1 +
 .../blog/2017/02/01/graduation-media-recap.html|   1 +
 content/blog/2017/02/13/stateful-processing.html   |   1 +
 content/blog/2017/03/16/python-sdk-release.html|   1 +
 .../blog/2017/05/17/beam-first-stable-release.html |   1 +
 content/blog/2017/08/16/splittable-do-fn.html  |   1 +
 content/blog/2017/08/28/timely-processing.html |   1 +
 content/blog/index.html|   1 +
 content/coming-soon.html   |   1 +
 content/contribute/contribution-guide/index.html   |   1 +
 content/contribute/design-principles/index.html|   1 +
 content/contribute/docker-images/index.html|   1 +
 content/contribute/index.html  |   1 +
 content/contribute/logos/index.html|   1 +
 content/contribute/maturity-model/index.html   |   1 +
 .../contribute/presentation-materials/index.html   |   1 +
 .../contribute/ptransform-style-guide/index.html   |   1 +
 content/contribute/release-guide/index.html|   1 +
 content/contribute/runner-guide/index.html |   1 +
 content/contribute/source-repository/index.html|   1 +
 content/contribute/team/index.html |   1 +
 content/contribute/testing/index.html  |   1 +
 content/contribute/work-in-progress/index.html |   1 +
 content/documentation/dsls/sql/index.html  |   1 +
 content/documentation/index.html   |   2 +
 content/documentation/io/authoring-java/index.html |   1 +
 .../documentation/io/authoring-overview/index.html |   1 +
 .../documentation/io/authoring-python/index.html   |   1 +
 .../documentation/io/built-in/hadoop/index.html|   1 +
 content/documentation/io/built-in/index.html   |   1 +
 content/documentation/io/contributing/index.html   |   1 +
 content/documentation/io/io-toc/index.html |   1 +
 content/documentation/io/testing/index.html|   1 +
 .../pipelines/create-your-pipeline/index.html  |   1 +
 .../pipelines/design-your-pipeline/index.html  |   1 +
 .../pipelines/test-your-pipeline/index.html|   1 +
 content/documentation/programming-guide/index.html |   1 +
 content/documentation/resources/index.html |   1 +
 content/documentation/runners/apex/index.html  |   1 +
 .../runners/capability-matrix/index.html   |   1 +
 content/documentation/runners/dataflow/index.html  |   1 +
 content/documentation/runners/direct/index.html|   1 +
 content/documentation/runners/flink/index.html |   1 +
 content/documentation/runners/gearpump/index.html  |   1 +
 content/documentation/runners/spark/index.html |   1 +
 .../documentation/sdks/java-extensions/index.html  |   1 +
 content/documentation/sdks/java/index.html |   1 +
 .../documentation/sdks/python-custom-io/index.html |   1 +
 .../sdks/python-pipeline-dependencies/index.html   |   1 +
 .../sdks/python-type-safety/index.html |   1 +
 content/documentation/sdks/python/index.html   |   1 +
 content/get-started/beam-overview/index.html   |   1 +
 content/get-started/downloads/index.html   |   1 +
 content/get-started/index.html |   1 +
 .../get-started/mobile-gaming-example/index.html   |   1 +
 content/get-started/quickstart-java/index.html |   1 +
 content/get-started/quickstart-py/index.html   |   1 +
 content/get-started/support/index.html |   1 +
 content/get-started/wordcount-examp

[beam-site] 02/04: Update with review feedback and new diagrams

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit fa4cf1e30f04933adf243353a2d13ffb0bde7915
Author: melissa 
AuthorDate: Wed Sep 27 17:23:42 2017 -0700

Update with review feedback and new diagrams
---
 src/documentation/execution-model.md   | 159 +
 src/images/execution_model_bundling.svg| 382 +++--
 .../execution_model_bundling_coupled_failure.svg   |  59 +++-
 src/images/execution_model_bundling_gantt.svg  |  55 ++-
 src/images/execution_model_bundling_gantt_max.svg  |  68 ++--
 src/images/execution_model_bundling_multi.svg  | 361 +--
 .../execution_model_bundling_multi_gantt.svg   |  60 ++--
 src/images/execution_model_failure_retry.svg   |  68 ++--
 8 files changed, 981 insertions(+), 231 deletions(-)

diff --git a/src/documentation/execution-model.md 
b/src/documentation/execution-model.md
index a8ebee0..e09fe56 100644
--- a/src/documentation/execution-model.md
+++ b/src/documentation/execution-model.md
@@ -6,13 +6,11 @@ permalink: /documentation/execution-model/
 
 # Apache Beam Execution Model
 
-The Beam model allows runners to execute your pipeline in different ways.
-Depending on the runner’s choices, you may observe various effects as a result.
-This page describes the effects of these choices so you can better understand
-how Beam pipelines execute.
+The Beam model allows runners to execute your pipeline in different ways. You
+may observe various effects as a result of the runner’s choices. This page
+describes these effects so you can better understand how Beam pipelines 
execute.
 
-**Table of Contents:**
-* TOC
+* toc
 {:toc}
 
 ## Processing of elements
@@ -22,41 +20,39 @@ most expensive operations in a distributed execution of 
your pipeline. Avoiding
 this serialization may require re-processing elements after failures or may
 limit the distribution of output to other machines.
 
-The runner processes elements on many machines and may serialize elements in
-between machines for other communication and persistence reasons.
-
 ### Serialization and communication
 
-The runner may serialize elements for communication or persistence.
+The runner may serialize elements between machines for communication purposes
+and for other reasons such as persistence. For example, a runner may serialize
+and persist elements in these two situations.
+
+1. When used as part of a stateful `DoFn`, the runner may persist values to 
some
+   state mechanism.
+1. When committing the results of processing, the runner may persist the 
outputs
+   as a checkpoint.
 
-The runner may decide to transfer elements between transforms in a variety of
+A runner may decide to transfer elements between transforms in a variety of
 ways, such as:
 
 1.  Routing elements to a worker for processing as part of a grouping 
operation.
-This may involve serializing elements and sorting them by their key.
+This may involve serializing elements and grouping or sorting them by their
+key.
 1.  Redistributing elements between workers to adjust parallelism. This may
 involve serializing elements and communicating them to other workers.
 1.  Using the elements in a side input to a `ParDo`. This may require
 serializing the elements and broadcasting them to all the workers executing
 the `ParDo`.
 1.  Passing elements between transforms that are running on the same worker.
-This may avoid having to serialize the elements, and instead just passing
-them in memory.
-
-Additionally, the runner may serialize and persist elements for other reasons:
-
-1. When used as part of a Stateful `DoFn`, the runner may persist values to 
some
-   state mechanism.
-1. When committing the results of processing, the runner may persist the 
outputs
-   as a checkpoint.
+This may allow the runner to avoid serializing elements; instead, the 
runner
+can just pass the elements in memory.
 
 ### Bundling and persistence
 
-Beam pipelines often focus on ["embarassingly 
parallel"](https://en.wikipedia.org/wiki/Embarrassingly_parallel)
-problems.  Because of this, the APIs emphasize processing elements in parallel,
-which makes it difficult to express things like "assign a sequence number to
-each element in a PCollection." This is intentional since such algorithms are
-much more likely to suffer from scalability problems.
+Beam pipelines often focus on "[embarassingly 
parallel](https://en.wikipedia.org/wiki/embarrassingly_parallel)"
+problems. Because of this, the APIs emphasize processing elements in parallel,
+which makes it difficult to express actions like "assign a sequence number to
+each element in a PCollection". This is intentional as such algorithms are much
+more likely to suffer from scalability problems.
 
 Processing all elements in parallel also has some drawbacks. Specifically, it
 makes

[beam-site] 03/04: More updates from review feedback

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 708bd587b45934e52a12a282b107cb14c4b45171
Author: melissa 
AuthorDate: Thu Sep 28 18:19:12 2017 -0700

More updates from review feedback
---
 src/documentation/execution-model.md | 19 ++-
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/documentation/execution-model.md 
b/src/documentation/execution-model.md
index e09fe56..049c7f3 100644
--- a/src/documentation/execution-model.md
+++ b/src/documentation/execution-model.md
@@ -22,14 +22,8 @@ limit the distribution of output to other machines.
 
 ### Serialization and communication
 
-The runner may serialize elements between machines for communication purposes
-and for other reasons such as persistence. For example, a runner may serialize
-and persist elements in these two situations.
-
-1. When used as part of a stateful `DoFn`, the runner may persist values to 
some
-   state mechanism.
-1. When committing the results of processing, the runner may persist the 
outputs
-   as a checkpoint.
+The runner might serialize elements between machines for communication purposes
+and for other reasons such as persistence.
 
 A runner may decide to transfer elements between transforms in a variety of
 ways, such as:
@@ -46,6 +40,13 @@ ways, such as:
 This may allow the runner to avoid serializing elements; instead, the 
runner
 can just pass the elements in memory.
 
+Some situations where the runner may serialize and persist elements are:
+
+1. When used as part of a stateful `DoFn`, the runner may persist values to 
some
+   state mechanism.
+1. When committing the results of processing, the runner may persist the 
outputs
+   as a checkpoint.
+
 ### Bundling and persistence
 
 Beam pipelines often focus on "[embarassingly 
parallel](https://en.wikipedia.org/wiki/embarrassingly_parallel)"
@@ -165,7 +166,7 @@ down and abandoned.
 If a failure to process an element in `ParDo2` causes `ParDo1` to re-execute,
 these two steps are said to be _co-failing_.
 
-For this example, we will use the dependently parallel `ParDo`s from figure 4.
+For this example, we will use the two `ParDo`s from figure 4.
 
 In figure 7, worker two successfully executes `ParDo1` on all elements in 
bundle
 B. However, the worker fails to process an element in bundle D, so `ParDo2`

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" .

[beam-site] branch mergebot updated (5faa02e -> 7a1740b)

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a change to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git.


from 5faa02e  This closes #328
 add 4052af2  Prepare repository for deployment.
 new 0b495f8  Add new execution model page and associated images
 new fa4cf1e  Update with review feedback and new diagrams
 new 708bd58  More updates from review feedback
 new 7a1740b  This closes #326

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 content/documentation/runners/dataflow/index.html  |   5 +-
 src/_includes/header.html  |   1 +
 src/documentation/execution-model.md   | 191 +++
 src/documentation/index.md |   9 +-
 src/images/execution_model_bundling.svg| 357 +
 .../execution_model_bundling_coupled_failure.svg   |  59 
 src/images/execution_model_bundling_gantt.svg  |  50 +++
 src/images/execution_model_bundling_gantt_max.svg  |  67 
 src/images/execution_model_bundling_multi.svg  | 336 +++
 .../execution_model_bundling_multi_gantt.svg   |  52 +++
 src/images/execution_model_failure_retry.svg   |  56 
 11 files changed, 1178 insertions(+), 5 deletions(-)
 create mode 100644 src/documentation/execution-model.md
 create mode 100644 src/images/execution_model_bundling.svg
 create mode 100644 src/images/execution_model_bundling_coupled_failure.svg
 create mode 100644 src/images/execution_model_bundling_gantt.svg
 create mode 100644 src/images/execution_model_bundling_gantt_max.svg
 create mode 100644 src/images/execution_model_bundling_multi.svg
 create mode 100644 src/images/execution_model_bundling_multi_gantt.svg
 create mode 100644 src/images/execution_model_failure_retry.svg

-- 
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" '].

[beam-site] 01/04: Add new execution model page and associated images

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 0b495f8c508389fed8a9f467104bc03b15d84b83
Author: melissa 
AuthorDate: Mon Sep 25 13:52:15 2017 -0700

Add new execution model page and associated images
---
 src/_includes/header.html  |   1 +
 src/documentation/execution-model.md   | 163 +
 src/documentation/index.md |   9 +-
 src/images/execution_model_bundling.svg|  33 +
 .../execution_model_bundling_coupled_failure.svg   |  38 +
 src/images/execution_model_bundling_gantt.svg  |  33 +
 src/images/execution_model_bundling_gantt_max.svg  |  45 ++
 src/images/execution_model_bundling_multi.svg  |  35 +
 .../execution_model_bundling_multi_gantt.svg   |  34 +
 src/images/execution_model_failure_retry.svg   |  36 +
 10 files changed, 423 insertions(+), 4 deletions(-)

diff --git a/src/_includes/header.html b/src/_includes/header.html
index 839b4ea..1981fb1 100644
--- a/src/_includes/header.html
+++ b/src/_includes/header.html
@@ -35,6 +35,7 @@
 
 Beam Concepts
 Programming Guide
+Execution Model
 Additional Resources
 
 Pipeline Fundamentals
diff --git a/src/documentation/execution-model.md 
b/src/documentation/execution-model.md
new file mode 100644
index 000..a8ebee0
--- /dev/null
+++ b/src/documentation/execution-model.md
@@ -0,0 +1,163 @@
+---
+layout: default
+title: "Beam Execution Model"
+permalink: /documentation/execution-model/
+---
+
+# Apache Beam Execution Model
+
+The Beam model allows runners to execute your pipeline in different ways.
+Depending on the runner’s choices, you may observe various effects as a result.
+This page describes the effects of these choices so you can better understand
+how Beam pipelines execute.
+
+**Table of Contents:**
+* TOC
+{:toc}
+
+## Processing of elements
+
+The serialization and communication of elements between machines is one of the
+most expensive operations in a distributed execution of your pipeline. Avoiding
+this serialization may require re-processing elements after failures or may
+limit the distribution of output to other machines.
+
+The runner processes elements on many machines and may serialize elements in
+between machines for other communication and persistence reasons.
+
+### Serialization and communication
+
+The runner may serialize elements for communication or persistence.
+
+The runner may decide to transfer elements between transforms in a variety of
+ways, such as:
+
+1.  Routing elements to a worker for processing as part of a grouping 
operation.
+This may involve serializing elements and sorting them by their key.
+1.  Redistributing elements between workers to adjust parallelism. This may
+involve serializing elements and communicating them to other workers.
+1.  Using the elements in a side input to a `ParDo`. This may require
+serializing the elements and broadcasting them to all the workers executing
+the `ParDo`.
+1.  Passing elements between transforms that are running on the same worker.
+This may avoid having to serialize the elements, and instead just passing
+them in memory.
+
+Additionally, the runner may serialize and persist elements for other reasons:
+
+1. When used as part of a Stateful `DoFn`, the runner may persist values to 
some
+   state mechanism.
+1. When committing the results of processing, the runner may persist the 
outputs
+   as a checkpoint.
+
+### Bundling and persistence
+
+Beam pipelines often focus on ["embarassingly 
parallel"](https://en.wikipedia.org/wiki/Embarrassingly_parallel)
+problems.  Because of this, the APIs emphasize processing elements in parallel,
+which makes it difficult to express things like "assign a sequence number to
+each element in a PCollection." This is intentional since such algorithms are
+much more likely to suffer from scalability problems.
+
+Processing all elements in parallel also has some drawbacks. Specifically, it
+makes it impossible to batch any operations, such as writing elements to a sink
+or checkpointing progress during processing.
+
+Instead of processing all elements simultaneously, the elements in a
+`PCollection` are processed in _bundles_. The division of the collection into
+bundles is arbitrary and selected by the runner. This allows the runner to
+choose an appropriate middle-ground between persisting results after every
+element, and having to retry everything if there is a failure.
+
+## Failures and parallelism within and between transforms
+
+In this section, we discuss how elements in the input collection are processed
+in parallel, and how transforms are retried when failures occur.
+
+### Data-parallelism within one transform {#data-parallelism}
+
+The bundli

[beam-site] 04/04: This closes #326

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 7a1740b8f7a148d0a5dcdee4d217afdfd2a49017
Merge: 4052af2 708bd58
Author: Mergebot 
AuthorDate: Fri Sep 29 01:37:40 2017 +

This closes #326

 src/_includes/header.html  |   1 +
 src/documentation/execution-model.md   | 191 +++
 src/documentation/index.md |   9 +-
 src/images/execution_model_bundling.svg| 357 +
 .../execution_model_bundling_coupled_failure.svg   |  59 
 src/images/execution_model_bundling_gantt.svg  |  50 +++
 src/images/execution_model_bundling_gantt_max.svg  |  67 
 src/images/execution_model_bundling_multi.svg  | 336 +++
 .../execution_model_bundling_multi_gantt.svg   |  52 +++
 src/images/execution_model_failure_retry.svg   |  56 
 10 files changed, 1174 insertions(+), 4 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" .

Jenkins build is back to normal : beam_PerformanceTests_Python #385

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Flink #3953

[Apache Jenkins Server]

See 

[jira] [Closed] (BEAM-2986) Support reading avro GenericRecords with BigQueryIO

[Eugene Kirpichov (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-2986?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Eugene Kirpichov closed BEAM-2986.
--
   Resolution: Fixed
Fix Version/s: 2.2.0

> Support reading avro GenericRecords with BigQueryIO
> ---
>
> Key: BEAM-2986
> URL: https://issues.apache.org/jira/browse/BEAM-2986
> Project: Beam
>  Issue Type: New Feature
>  Components: sdk-java-gcp
>Reporter: Steve Niemitz
>Assignee: Chamikara Jayalath
> Fix For: 2.2.0
>
>
> It would be nice to expose the avro records from BigQuery exports directly, 
> rather than converting/serializing them to TableRow objects.
> The interface could be similar to AvroIO.parseGenericRecords, allowing a user 
> to pass in a custom parseFn to the reader.
> There's a thread on the dev@ mailing this w/ discussion:
> https://lists.apache.org/thread.html/a045726a0c8d7412f3556ee3aac881920afb040180f42112b393c68e@%3Cdev.beam.apache.org%3E



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Flink #3952

[Apache Jenkins Server]

See 

[jira] [Commented] (BEAM-2986) Support reading avro GenericRecords with BigQueryIO

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-2986?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16185113#comment-16185113
 ] 

ASF GitHub Bot commented on BEAM-2986:
--

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3894


> Support reading avro GenericRecords with BigQueryIO
> ---
>
> Key: BEAM-2986
> URL: https://issues.apache.org/jira/browse/BEAM-2986
> Project: Beam
>  Issue Type: New Feature
>  Components: sdk-java-gcp
>Reporter: Steve Niemitz
>Assignee: Chamikara Jayalath
>
> It would be nice to expose the avro records from BigQuery exports directly, 
> rather than converting/serializing them to TableRow objects.
> The interface could be similar to AvroIO.parseGenericRecords, allowing a user 
> to pass in a custom parseFn to the reader.
> There's a thread on the dev@ mailing this w/ discussion:
> https://lists.apache.org/thread.html/a045726a0c8d7412f3556ee3aac881920afb040180f42112b393c68e@%3Cdev.beam.apache.org%3E



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[1/2] beam git commit: Support for using raw avro records from BigQuery

[jkff]

Repository: beam
Updated Branches:
  refs/heads/master dc597873e -> 6be9e0bb2


Support for using raw avro records from BigQuery


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c98c8049
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c98c8049
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c98c8049

Branch: refs/heads/master
Commit: c98c8049fa3607c4cadbabffeba4bde493225540
Parents: dc59787
Author: steve 
Authored: Fri Sep 22 11:07:07 2017 -0400
Committer: Eugene Kirpichov 
Committed: Thu Sep 28 16:05:39 2017 -0700

--
 .../examples/cookbook/BigQueryTornadoes.java|   2 +-
 .../cookbook/CombinePerKeyExamples.java |   2 +-
 .../beam/examples/cookbook/FilterExamples.java  |   2 +-
 .../beam/examples/cookbook/JoinExamples.java|   6 +-
 .../examples/cookbook/MaxPerKeyExamples.java|   2 +-
 .../org/apache/beam/sdk/io/package-info.java|   2 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java| 407 ++-
 .../io/gcp/bigquery/BigQueryQuerySource.java|  22 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java |  69 ++--
 .../io/gcp/bigquery/BigQueryTableSource.java|  19 +-
 .../sdk/io/gcp/bigquery/SchemaAndRecord.java|  43 ++
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java  |   2 +
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 100 +++--
 13 files changed, 498 insertions(+), 180 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/c98c8049/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
index 07a3edd..df9ff5a 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
@@ -156,7 +156,7 @@ public class BigQueryTornadoes {
 fields.add(new 
TableFieldSchema().setName("tornado_count").setType("INTEGER"));
 TableSchema schema = new TableSchema().setFields(fields);
 
-p.apply(BigQueryIO.read().from(options.getInput()))
+p.apply(BigQueryIO.readTableRows().from(options.getInput()))
  .apply(new CountTornadoes())
  .apply(BigQueryIO.writeTableRows()
  .to(options.getOutput())

http://git-wip-us.apache.org/repos/asf/beam/blob/c98c8049/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
index 693f0c4..1e91aec 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
@@ -195,7 +195,7 @@ public class CombinePerKeyExamples {
 fields.add(new TableFieldSchema().setName("all_plays").setType("STRING"));
 TableSchema schema = new TableSchema().setFields(fields);
 
-p.apply(BigQueryIO.read().from(options.getInput()))
+p.apply(BigQueryIO.readTableRows().from(options.getInput()))
  .apply(new PlaysForWord())
  .apply(BigQueryIO.writeTableRows()
 .to(options.getOutput())

http://git-wip-us.apache.org/repos/asf/beam/blob/c98c8049/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
index fed9db7..a4fe425 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
@@ -237,7 +237,7 @@ public class FilterExamples {
 
 TableSchema schema = buildWeatherSchemaProjection();
 
-p.apply(BigQueryIO.read().from(options.getInput()))
+p.apply(BigQueryIO.readTableRows().from(options.getInput()))
  .apply(ParDo.of(new ProjectionFn()))
  .apply(new BelowGlobalMean(options.getMonthFilter()))
  .apply(BigQueryIO.writeTableRows()

http://git-wip-us.apache.org/repos/asf/beam/blob/c98c8049/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java
 
b/exampl

[GitHub] beam pull request #3894: [BEAM-2986] Support for reading avro GenericRecords...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3894


---

[2/2] beam git commit: This closes #3894: [BEAM-2986] Support for reading avro GenericRecords from BigQuery

[jkff]

This closes #3894: [BEAM-2986] Support for reading avro GenericRecords from 
BigQuery


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6be9e0bb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6be9e0bb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6be9e0bb

Branch: refs/heads/master
Commit: 6be9e0bb29803651a45845382e2a9badea1a37b7
Parents: dc59787 c98c804
Author: Eugene Kirpichov 
Authored: Thu Sep 28 16:40:54 2017 -0700
Committer: Eugene Kirpichov 
Committed: Thu Sep 28 16:40:54 2017 -0700

--
 .../examples/cookbook/BigQueryTornadoes.java|   2 +-
 .../cookbook/CombinePerKeyExamples.java |   2 +-
 .../beam/examples/cookbook/FilterExamples.java  |   2 +-
 .../beam/examples/cookbook/JoinExamples.java|   6 +-
 .../examples/cookbook/MaxPerKeyExamples.java|   2 +-
 .../org/apache/beam/sdk/io/package-info.java|   2 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java| 407 ++-
 .../io/gcp/bigquery/BigQueryQuerySource.java|  22 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java |  69 ++--
 .../io/gcp/bigquery/BigQueryTableSource.java|  19 +-
 .../sdk/io/gcp/bigquery/SchemaAndRecord.java|  43 ++
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java  |   2 +
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 100 +++--
 13 files changed, 498 insertions(+), 180 deletions(-)
--

Jenkins build is still unstable: beam_PostCommit_Java_MavenInstall #4903

[Apache Jenkins Server]

See 

[beam-site] branch asf-site updated (6c9f49e -> 4052af2)

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a change to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git.


from 6c9f49e  Prepare repository for deployment.
 add 1e92c93  Add Cloud Resource Manager API to the required API list
 add 5faa02e  This closes #328
 new 4052af2  Prepare repository for deployment.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 content/documentation/runners/dataflow/index.html | 5 -
 src/documentation/runners/dataflow.md | 5 -
 2 files changed, 8 insertions(+), 2 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" '].

[beam-site] 01/01: Prepare repository for deployment.

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 4052af26d2723e588ed10fd91155998e053efa01
Author: Mergebot 
AuthorDate: Thu Sep 28 22:56:55 2017 +

Prepare repository for deployment.
---
 content/documentation/runners/dataflow/index.html | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/content/documentation/runners/dataflow/index.html 
b/content/documentation/runners/dataflow/index.html
index 108b890..8d0ed62 100644
--- a/content/documentation/runners/dataflow/index.html
+++ b/content/documentation/runners/dataflow/index.html
@@ -176,7 +176,10 @@
 Enable billing for your project.
   
   
-Enable required Google Cloud APIs: Cloud Dataflow, Compute Engine, 
Stackdriver Logging, Cloud Storage, and Cloud Storage JSON. You may need to 
enable additional APIs (such as BigQuery, Cloud Pub/Sub, or Cloud Datastore) if 
you use them in your pipeline code.
+Enable the required Google Cloud APIs: Cloud Dataflow, Compute Engine,
+Stackdriver Logging, Cloud Storage, Cloud Storage JSON, and Cloud Resource
+Manager. You may need to enable additional APIs (such as BigQuery, Cloud
+Pub/Sub, or Cloud Datastore) if you use them in your pipeline code.
   
   
 Install the Google Cloud SDK.

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" .

[beam-site] branch mergebot updated (15be8cf -> 5faa02e)

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a change to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git.


from 15be8cf  This closes #327
 add 6c9f49e  Prepare repository for deployment.
 new 1e92c93  Add Cloud Resource Manager API to the required API list
 new 5faa02e  This closes #328

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 content/blog/2017/02/13/stateful-processing.html | 6 +++---
 content/feed.xml | 6 +++---
 src/documentation/runners/dataflow.md| 5 -
 3 files changed, 10 insertions(+), 7 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" '].

[beam-site] 02/02: This closes #328

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 5faa02eac8ef6b138191a552cc9d2e55ef8ded9f
Merge: 6c9f49e 1e92c93
Author: Mergebot 
AuthorDate: Thu Sep 28 22:53:07 2017 +

This closes #328

 src/documentation/runners/dataflow.md | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" .

[beam-site] 01/02: Add Cloud Resource Manager API to the required API list

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 1e92c932e298649dbaec785f6bb14a65b10903b0
Author: melissa 
AuthorDate: Thu Sep 28 15:35:35 2017 -0700

Add Cloud Resource Manager API to the required API list
---
 src/documentation/runners/dataflow.md | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/documentation/runners/dataflow.md 
b/src/documentation/runners/dataflow.md
index 4960ce9..8ef3ba2 100644
--- a/src/documentation/runners/dataflow.md
+++ b/src/documentation/runners/dataflow.md
@@ -31,7 +31,10 @@ To use the Cloud Dataflow Runner, you must complete the 
following setup:
 
 2. Enable billing for your project.
 
-3. Enable required Google Cloud APIs: Cloud Dataflow, Compute Engine, 
Stackdriver Logging, Cloud Storage, and Cloud Storage JSON. You may need to 
enable additional APIs (such as BigQuery, Cloud Pub/Sub, or Cloud Datastore) if 
you use them in your pipeline code.
+3. Enable the required Google Cloud APIs: Cloud Dataflow, Compute Engine,
+   Stackdriver Logging, Cloud Storage, Cloud Storage JSON, and Cloud Resource
+   Manager. You may need to enable additional APIs (such as BigQuery, Cloud
+   Pub/Sub, or Cloud Datastore) if you use them in your pipeline code.
 
 4. Install the Google Cloud SDK.
 

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" .

[jira] [Commented] (BEAM-2596) Break up Jenkins PreCommit into individual steps.

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-2596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16185029#comment-16185029
 ] 

ASF GitHub Bot commented on BEAM-2596:
--

GitHub user jasonkuster opened a pull request:

https://github.com/apache/beam/pull/3918

[BEAM-2596] Pipeline Job Choose Branch

Allow users to skip Java or Python branch via a comment. Also use try/catch 
blocks to stop non-blocking branches from stopping the build.

Signed-off-by: Jason Kuster 

Follow this checklist to help us incorporate your contribution quickly and 
easily:

 - [x] Make sure there is a [JIRA 
issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the 
change (usually before you start working on it).  Trivial changes like typos do 
not require a JIRA issue.  Your pull request should address just this issue, 
without pulling in other changes.
 - [x] Each commit in the pull request should have a meaningful subject 
line and body.
 - [x] Format the pull request title like `[BEAM-XXX] Fixes bug in 
ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA 
issue.
 - [x] Write a pull request description that is detailed enough to 
understand what the pull request does, how, and why.
 - [x] Run `mvn clean verify` to make sure basic checks pass. A more 
thorough check will be performed on your pull request automatically.
 - [x] If this contribution is large, please file an Apache [Individual 
Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/jasonkuster/beam pipeline-only

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3918.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3918


commit 834517157264f90abd6466a041b6dd912cb10ad1
Author: Jason Kuster 
Date:   2017-09-28T22:34:23Z

Allow users to skip Java or Python branch via a comment. Also use try/catch 
blocks to stop non-blocking branches from stopping the build.

Signed-off-by: Jason Kuster 




> Break up Jenkins PreCommit into individual steps.
> -
>
> Key: BEAM-2596
> URL: https://issues.apache.org/jira/browse/BEAM-2596
> Project: Beam
>  Issue Type: New Feature
>  Components: build-system, testing
>Reporter: Jason Kuster
>Assignee: Jason Kuster
>




--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[GitHub] beam pull request #3918: [BEAM-2596] Pipeline Job Choose Branch

[jasonkuster]

GitHub user jasonkuster opened a pull request:

https://github.com/apache/beam/pull/3918

[BEAM-2596] Pipeline Job Choose Branch

Allow users to skip Java or Python branch via a comment. Also use try/catch 
blocks to stop non-blocking branches from stopping the build.

Signed-off-by: Jason Kuster 

Follow this checklist to help us incorporate your contribution quickly and 
easily:

 - [x] Make sure there is a [JIRA 
issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the 
change (usually before you start working on it).  Trivial changes like typos do 
not require a JIRA issue.  Your pull request should address just this issue, 
without pulling in other changes.
 - [x] Each commit in the pull request should have a meaningful subject 
line and body.
 - [x] Format the pull request title like `[BEAM-XXX] Fixes bug in 
ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA 
issue.
 - [x] Write a pull request description that is detailed enough to 
understand what the pull request does, how, and why.
 - [x] Run `mvn clean verify` to make sure basic checks pass. A more 
thorough check will be performed on your pull request automatically.
 - [x] If this contribution is large, please file an Apache [Individual 
Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/jasonkuster/beam pipeline-only

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3918.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3918


commit 834517157264f90abd6466a041b6dd912cb10ad1
Author: Jason Kuster 
Date:   2017-09-28T22:34:23Z

Allow users to skip Java or Python branch via a comment. Also use try/catch 
blocks to stop non-blocking branches from stopping the build.

Signed-off-by: Jason Kuster 




---

[jira] [Commented] (BEAM-2998) add IT test for SQL

[Xu Mingmin (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-2998?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16185018#comment-16185018
 ] 

Xu Mingmin commented on BEAM-2998:
--

seems CI job cannot run IT tests on shaded libraries, true?

> add IT test for SQL
> ---
>
> Key: BEAM-2998
> URL: https://issues.apache.org/jira/browse/BEAM-2998
> Project: Beam
>  Issue Type: Test
>  Components: dsl-sql, testing
>Reporter: Xu Mingmin
>
> Add IT test for SQL module
> https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
>  is the base example.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Flink #3951

[Apache Jenkins Server]

See 

Jenkins build is back to normal : beam_PostCommit_Java_ValidatesRunner_Apex #2490

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Flink #3950

[Apache Jenkins Server]

See 

[jira] [Commented] (BEAM-2596) Break up Jenkins PreCommit into individual steps.

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-2596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16184962#comment-16184962
 ] 

ASF GitHub Bot commented on BEAM-2596:
--

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3916


> Break up Jenkins PreCommit into individual steps.
> -
>
> Key: BEAM-2596
> URL: https://issues.apache.org/jira/browse/BEAM-2596
> Project: Beam
>  Issue Type: New Feature
>  Components: build-system, testing
>Reporter: Jason Kuster
>Assignee: Jason Kuster
>




--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[GitHub] beam pull request #3916: [BEAM-2596] Updated Pipeline job for Jenkins PreCom...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3916


---

[1/2] beam git commit: Closes #3916

[robertwb]

Repository: beam
Updated Branches:
  refs/heads/master 0724f1c5f -> dc597873e


Closes #3916


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/dc597873
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/dc597873
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/dc597873

Branch: refs/heads/master
Commit: dc597873eda4c5b5c717fecf3be14361bacde88a
Parents: 0724f1c d4323e9
Author: Robert Bradshaw 
Authored: Thu Sep 28 14:53:29 2017 -0700
Committer: Robert Bradshaw 
Committed: Thu Sep 28 14:53:29 2017 -0700

--
 .test-infra/jenkins/PreCommit_Pipeline.groovy   |  89 +
 .../jenkins/common_job_properties.groovy| 185 ++-
 .test-infra/jenkins/job_beam_Java_Build.groovy  |  82 
 .../jenkins/job_beam_Java_CodeHealth.groovy |  39 
 .../job_beam_Java_IntegrationTest.groovy|  63 +++
 .../jenkins/job_beam_Java_UnitTest.groovy   |  49 +
 .../jenkins/job_beam_PreCommit_Pipeline.groovy  |  84 +
 .../jenkins/job_beam_Python_UnitTest.groovy |  40 
 8 files changed, 584 insertions(+), 47 deletions(-)
--

[2/2] beam git commit: Updated Pipeline job for Jenkins PreCommit

[robertwb]

Updated Pipeline job for Jenkins PreCommit

Signed-off-by: Jason Kuster 


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d4323e98
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d4323e98
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d4323e98

Branch: refs/heads/master
Commit: d4323e98c0e287236169627e972c88e5bb1f1d53
Parents: 0724f1c
Author: Jason Kuster 
Authored: Wed Jun 28 16:22:52 2017 -0700
Committer: Robert Bradshaw 
Committed: Thu Sep 28 14:53:29 2017 -0700

--
 .test-infra/jenkins/PreCommit_Pipeline.groovy   |  89 +
 .../jenkins/common_job_properties.groovy| 185 ++-
 .test-infra/jenkins/job_beam_Java_Build.groovy  |  82 
 .../jenkins/job_beam_Java_CodeHealth.groovy |  39 
 .../job_beam_Java_IntegrationTest.groovy|  63 +++
 .../jenkins/job_beam_Java_UnitTest.groovy   |  49 +
 .../jenkins/job_beam_PreCommit_Pipeline.groovy  |  84 +
 .../jenkins/job_beam_Python_UnitTest.groovy |  40 
 8 files changed, 584 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/PreCommit_Pipeline.groovy
--
diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy 
b/.test-infra/jenkins/PreCommit_Pipeline.groovy
new file mode 100644
index 000..9abf39d
--- /dev/null
+++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy
@@ -0,0 +1,89 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import hudson.model.Result
+
+int NO_BUILD = -1
+
+// These are args for the GitHub Pull Request Builder (ghprb) Plugin. 
Providing these arguments is
+// necessary due to a bug in the ghprb plugin where environment variables are 
not correctly passed
+// to jobs downstream of a Pipeline job.
+// Tracked by https://github.com/jenkinsci/ghprb-plugin/issues/572.
+List ghprbArgs = [
+string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"),
+string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"),
+string(name: 'ghprbPullId', value: "${ghprbPullId}")
+]
+
+// This argument is the commit at which to build.
+List commitArg = [string(name: 'sha1', value: 
"origin/pr/${ghprbPullId}/head")]
+
+int javaBuildNum = NO_BUILD
+
+// This (and the below) define "Stages" of a pipeline. These stages run 
serially, and inside can
+// have "parallel" blocks which execute several work steps concurrently. This 
work is limited to
+// simple operations -- more complicated operations need to be performed on an 
actual node. In this
+// case we are using the pipeline to trigger downstream builds.
+stage('Build') {
+parallel (
+java: {
+def javaBuild = build job: 'beam_Java_Build', parameters: 
commitArg + ghprbArgs
+if(javaBuild.getResult() == Result.SUCCESS.toString()) {
+javaBuildNum = javaBuild.getNumber()
+}
+},
+python_unit: { // Python doesn't have a build phase, so we include 
this here.
+build job: 'beam_Python_UnitTest', parameters: commitArg + 
ghprbArgs
+}
+)
+}
+
+// This argument is provided to downstream jobs so they know from which build 
to pull artifacts.
+javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")]
+javaUnitPassed = false
+
+stage('Unit Test / Code Health') {
+parallel (
+java_unit: {
+if(javaBuildNum != NO_BUILD) {
+def javaTest = build job: 'beam_Java_UnitTest', parameters: 
javaBuildArg + ghprbArgs
+if(javaTest.getResult() == Result.SUCCESS.toString()) {
+javaUnitPassed = true
+}
+}
+},
+java_codehealth: {
+if(javaBuildNum != NO_BUILD) {
+build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + 
ghprbArgs
+}
+}
+)
+}
+
+stage('Integration Test') {
+parallel (
+// Not gated on codehealth because codehealth s

[jira] [Commented] (BEAM-3001) Allow disabling validation on BigtableIO.write()

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-3001?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16184935#comment-16184935
 ] 

ASF GitHub Bot commented on BEAM-3001:
--

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3897


> Allow disabling validation on BigtableIO.write()
> 
>
> Key: BEAM-3001
> URL: https://issues.apache.org/jira/browse/BEAM-3001
> Project: Beam
>  Issue Type: Improvement
>  Components: sdk-java-gcp
>Reporter: Steve Niemitz
>Assignee: Chamikara Jayalath
> Fix For: 2.2.0
>
>
> Similar to BigQueryIO.Write.withoutValidation(), we should allow not 
> validating the destination table exists, since it may be created earlier in 
> the pipeline.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[jira] [Resolved] (BEAM-3001) Allow disabling validation on BigtableIO.write()

[Chamikara Jayalath (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-3001?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Chamikara Jayalath resolved BEAM-3001.
--
   Resolution: Fixed
Fix Version/s: 2.2.0

> Allow disabling validation on BigtableIO.write()
> 
>
> Key: BEAM-3001
> URL: https://issues.apache.org/jira/browse/BEAM-3001
> Project: Beam
>  Issue Type: Improvement
>  Components: sdk-java-gcp
>Reporter: Steve Niemitz
>Assignee: Chamikara Jayalath
> Fix For: 2.2.0
>
>
> Similar to BigQueryIO.Write.withoutValidation(), we should allow not 
> validating the destination table exists, since it may be created earlier in 
> the pipeline.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[GitHub] beam pull request #3897: [BEAM-3001] Option to disable validation of Bigtabl...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3897


---

[jira] [Commented] (BEAM-3001) Allow disabling validation on BigtableIO.write()

[Chamikara Jayalath (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-3001?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16184936#comment-16184936
 ] 

Chamikara Jayalath commented on BEAM-3001:
--

cc: [~sduskis]

> Allow disabling validation on BigtableIO.write()
> 
>
> Key: BEAM-3001
> URL: https://issues.apache.org/jira/browse/BEAM-3001
> Project: Beam
>  Issue Type: Improvement
>  Components: sdk-java-gcp
>Reporter: Steve Niemitz
>Assignee: Chamikara Jayalath
> Fix For: 2.2.0
>
>
> Similar to BigQueryIO.Write.withoutValidation(), we should allow not 
> validating the destination table exists, since it may be created earlier in 
> the pipeline.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[2/2] beam git commit: This closes #3897

[chamikara]

This closes #3897


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0724f1c5
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0724f1c5
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0724f1c5

Branch: refs/heads/master
Commit: 0724f1c5f73ca7062226469ac34a2da970ad67e8
Parents: 3436053 9cea17e
Author: chamik...@google.com 
Authored: Thu Sep 28 14:38:00 2017 -0700
Committer: chamik...@google.com 
Committed: Thu Sep 28 14:38:00 2017 -0700

--
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java| 60 ++--
 .../sdk/io/gcp/bigtable/BigtableIOTest.java | 26 +
 2 files changed, 70 insertions(+), 16 deletions(-)
--

[1/2] beam git commit: Option to disable validation of BigtableIO.write target table

[chamikara]

Repository: beam
Updated Branches:
  refs/heads/master 34360537f -> 0724f1c5f


Option to disable validation of BigtableIO.write target table


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9cea17eb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9cea17eb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9cea17eb

Branch: refs/heads/master
Commit: 9cea17ebd00feb5280efc6480643f7b55a4ec273
Parents: 3436053
Author: steve 
Authored: Mon Sep 25 14:21:54 2017 -0400
Committer: chamik...@google.com 
Committed: Thu Sep 28 14:37:52 2017 -0700

--
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java| 60 ++--
 .../sdk/io/gcp/bigtable/BigtableIOTest.java | 26 +
 2 files changed, 70 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/9cea17eb/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
--
diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index fd15240..252f6c5 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -190,7 +190,10 @@ public class BigtableIO {
*/
   @Experimental
   public static Read read() {
-return new 
AutoValue_BigtableIO_Read.Builder().setKeyRange(ByteKeyRange.ALL_KEYS).setTableId("")
+return new AutoValue_BigtableIO_Read.Builder()
+.setKeyRange(ByteKeyRange.ALL_KEYS)
+.setTableId("")
+.setValidate(true)
 .build();
   }
 
@@ -203,7 +206,10 @@ public class BigtableIO {
*/
   @Experimental
   public static Write write() {
-return new AutoValue_BigtableIO_Write.Builder().setTableId("").build();
+return new AutoValue_BigtableIO_Write.Builder()
+.setTableId("")
+.setValidate(true)
+.build();
   }
 
   /**
@@ -234,6 +240,8 @@ public class BigtableIO {
 @Nullable
 public abstract BigtableOptions getBigtableOptions();
 
+public abstract boolean getValidate();
+
 abstract Builder toBuilder();
 
 @AutoValue.Builder
@@ -249,6 +257,8 @@ public class BigtableIO {
 
   abstract Builder setBigtableService(BigtableService bigtableService);
 
+  abstract Builder setValidate(boolean validate);
+
   abstract Read build();
 }
 
@@ -316,6 +326,11 @@ public class BigtableIO {
   return toBuilder().setTableId(tableId).build();
 }
 
+/** Disables validation that the table being read from exists. */
+public Read withoutValidation() {
+  return toBuilder().setValidate(false).build();
+}
+
 @Override
 public PCollection expand(PBegin input) {
   checkArgument(getBigtableOptions() != null, "withBigtableOptions() is 
required");
@@ -332,13 +347,15 @@ public class BigtableIO {
 
 @Override
 public void validate(PipelineOptions options) {
-  try {
-checkArgument(
-getBigtableService(options).tableExists(getTableId()),
-"Table %s does not exist",
-getTableId());
-  } catch (IOException e) {
-LOG.warn("Error checking whether table {} exists; proceeding.", 
getTableId(), e);
+  if (getValidate()) {
+try {
+  checkArgument(
+  getBigtableService(options).tableExists(getTableId()),
+  "Table %s does not exist",
+  getTableId());
+} catch (IOException e) {
+  LOG.warn("Error checking whether table {} exists; proceeding.", 
getTableId(), e);
+}
   }
 }
 
@@ -432,6 +449,8 @@ public class BigtableIO {
 @Nullable
 public abstract BigtableOptions getBigtableOptions();
 
+abstract boolean getValidate();
+
 abstract Builder toBuilder();
 
 @AutoValue.Builder
@@ -443,6 +462,8 @@ public class BigtableIO {
 
   abstract Builder setBigtableService(BigtableService bigtableService);
 
+  abstract Builder setValidate(boolean validate);
+
   abstract Write build();
 }
 
@@ -482,6 +503,11 @@ public class BigtableIO {
   return toBuilder().setBigtableOptions(optionsWithAgent).build();
 }
 
+/** Disables validation that the table being written to exists. */
+public Write withoutValidation() {
+  return toBuilder().setValidate(false).build();
+}
+
 /**
  * Returns a new {@link BigtableIO.Write} that will write to the specified 
table.
  *
@@ -509,13 +535,15 @@ public class BigtableIO {
 
 @Override
 public void validate(PipelineOptions options) {

[jira] [Updated] (BEAM-2998) add IT test for SQL

[Xu Mingmin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-2998?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Xu Mingmin updated BEAM-2998:
-
Component/s: testing

> add IT test for SQL
> ---
>
> Key: BEAM-2998
> URL: https://issues.apache.org/jira/browse/BEAM-2998
> Project: Beam
>  Issue Type: Test
>  Components: dsl-sql, testing
>Reporter: Xu Mingmin
>
> Add IT test for SQL module
> https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
>  is the base example.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[jira] [Commented] (BEAM-2984) Job submission too large with embedded Beam protos

[Luke Cwik (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-2984?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16184883#comment-16184883
 ] 

Luke Cwik commented on BEAM-2984:
-

Its additional information about the methodology used.

> Job submission too large with embedded Beam protos
> --
>
> Key: BEAM-2984
> URL: https://issues.apache.org/jira/browse/BEAM-2984
> Project: Beam
>  Issue Type: Bug
>  Components: runner-dataflow
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
>Priority: Blocker
> Fix For: 2.2.0
>
>
> Empirically, naively putting context around the {{DoFnInfo}} could cause a 
> blowup of 40%, which is too much and might cause jobs that were will under 
> API size limits to start to fail.
> There's a certain amount of wiggle room since it is hard to control the 
> submission size anyhow, but 40% is way too much.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

[beam-site] branch asf-site updated (82cd4ff -> 6c9f49e)

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a change to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git.


from 82cd4ff  This closes #325
 add 821b13b  Remove extraneous type variable from StateSpec in state blog 
post
 add 15be8cf  This closes #327
 new 6c9f49e  Prepare repository for deployment.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 content/blog/2017/02/13/stateful-processing.html | 6 +++---
 content/feed.xml | 6 +++---
 src/_posts/2017-02-13-stateful-processing.md | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" '].

[beam-site] 01/01: Prepare repository for deployment.

[mergebot-role]

This is an automated email from the ASF dual-hosted git repository.

mergebot-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git

commit 6c9f49ee5420818e719af453ffcfa5ac0d03963a
Author: Mergebot 
AuthorDate: Thu Sep 28 21:05:34 2017 +

Prepare repository for deployment.
---
 content/blog/2017/02/13/stateful-processing.html | 6 +++---
 content/feed.xml | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/content/blog/2017/02/13/stateful-processing.html 
b/content/blog/2017/02/13/stateful-processing.html
index 817176c..c927021 100644
--- a/content/blog/2017/02/13/stateful-processing.html
+++ b/content/blog/2017/02/13/stateful-processing.html
@@ -425,7 +425,7 @@ on a per key-and-window basis:
 
   // A state cell holding a single Integer per 
key+window
   @StateId("index")
-  private final StateSpec> indexSpec = 
+  private final StateSpec> indexSpec = 
   StateSpecs.value(VarIntCoder.of());
 
   @ProcessElement
@@ -620,11 +620,11 @@ only features I have already introduced:
 new DoFn, KV>() < [...]
 
   @StateId("model")
-  private final StateSpec> modelSpec =
+  private final StateSpec> modelSpec =
   StateSpecs.value(Model.coder());
 
   @StateId("previousPrediction")
-  private final StateSpec> previousPredictionSpec =
+  private final StateSpec> previousPredictionSpec =
   StateSpecs.value(Prediction.coder());
 
   @ProcessElement
diff --git a/content/feed.xml b/content/feed.xml
index 7cb78ab..0704f97 100644
--- a/content/feed.xml
+++ b/content/feed.xml
@@ -1540,7 +1540,7 @@ on a per key-and-window basis:

 
   // A state cell holding a single Integer 
per key+window
   @StateId("index")
-  private final StateSpec<Object, ValueState<Integerprivate final StateSpec<ValueState<Integer>> indexSpec  [...]
   StateSpecs.value(VarIntCoder.of());
 
   @ProcessElement
@@ -1735,11 +1735,11 @@ only features I have already introduced:

 
new DoFn<KV<UserId,  [...]
 
   @StateId("model")
-  private final StateSpec<Object, ValueState<Modelprivate final StateSpec<ValueState<Model>> modelSpec &l [...]
   StateSpecs.value(Model.coder());
 
   @StateId("previousPrediction")
-  private final StateSpec<Object, ValueState<Predictionprivate final StateSpec<ValueState<Prediction>> previousPredictionSp [...]
   StateSpecs