See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/1383/display/redirect>
------------------------------------------ [...truncated 1.88 KB...] [EnvInject] - Injecting as environment variables the properties content SPARK_LOCAL_IP=127.0.0.1 [EnvInject] - Variables injected successfully. [beam_PostCommit_Python_ValidatesRunner_Dataflow] $ /bin/bash -xe /tmp/jenkins2621267320739490253.sh + cd src + bash sdks/python/run_validatesrunner.sh # pip install --user installation location. LOCAL_PATH=$HOME/.local/bin/ # INFRA does not install virtualenv pip install virtualenv --user Requirement already satisfied: virtualenv in /home/jenkins/.local/lib/python2.7/site-packages (15.2.0) cheetah 2.4.4 requires Markdown>=2.0.1, which is not installed. apache-beam 2.5.0.dev0 requires hdfs<3.0.0,>=2.1.0, which is not installed. apache-beam 2.5.0.dev0 requires pytz>=2018.3, which is not installed. apache-beam 2.5.0.dev0 has requirement grpcio<2,>=1.8, but you'll have grpcio 1.4.0 which is incompatible. # Virtualenv for the rest of the script to run setup & e2e tests ${LOCAL_PATH}/virtualenv sdks/python New python executable in <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/bin/python> Installing setuptools, pip, wheel...done. . sdks/python/bin/activate # This file must be used with "source bin/activate" *from bash* # you cannot run it directly deactivate () { unset -f pydoc >/dev/null 2>&1 # reset old environment variables # ! [ -z ${VAR+_} ] returns true if VAR is declared at all if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then PATH="$_OLD_VIRTUAL_PATH" export PATH unset _OLD_VIRTUAL_PATH fi if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" export PYTHONHOME unset _OLD_VIRTUAL_PYTHONHOME fi # This should detect bash and zsh, which have a hash command that must # be called to get it to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then hash -r 2>/dev/null fi if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then PS1="$_OLD_VIRTUAL_PS1" export PS1 unset _OLD_VIRTUAL_PS1 fi unset VIRTUAL_ENV if [ ! "${1-}" = "nondestructive" ] ; then # Self destruct! unset -f deactivate fi } # unset irrelevant variables deactivate nondestructive VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python"> export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/bin:$PATH" export PATH # unset PYTHONHOME if set if ! [ -z "${PYTHONHOME+_}" ] ; then _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" unset PYTHONHOME fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then _OLD_VIRTUAL_PS1="$PS1" if [ "x" != x ] ; then PS1="$PS1" else PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1" fi export PS1 fi basename "$VIRTUAL_ENV" # Make sure to unalias pydoc if it's already there alias pydoc 2>/dev/null >/dev/null && unalias pydoc pydoc () { python -m pydoc "$@" } # This should detect bash and zsh, which have a hash command that must # be called to get it to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then hash -r 2>/dev/null fi cd sdks/python pip install -e .[gcp,test] Obtaining file://<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python> Collecting avro<2.0.0,>=1.8.1 (from apache-beam==2.5.0.dev0) Collecting crcmod<2.0,>=1.7 (from apache-beam==2.5.0.dev0) Collecting dill==0.2.6 (from apache-beam==2.5.0.dev0) Collecting grpcio<2,>=1.8 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/0d/54/b647a6323be6526be27b2c90bb042769f1a7a6e59bd1a5f2eeb795bfece4/grpcio-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.5.0.dev0) Collecting httplib2<0.10,>=0.8 (from apache-beam==2.5.0.dev0) Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl Collecting oauth2client<5,>=2.0.1 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/82/d8/3eab58811282ac7271a081ba5c0d4b875ce786ca68ce43e2a62ade32e9a8/oauth2client-4.1.2-py2.py3-none-any.whl Collecting protobuf<4,>=3.5.0.post1 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/9d/61/54c3a9cfde6ffe0ca6a1786ddb8874263f4ca32e7693ad383bd8cf935015/protobuf-3.5.2.post1-cp27-cp27mu-manylinux1_x86_64.whl Collecting pytz>=2018.3 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/dc/83/15f7833b70d3e067ca91467ca245bae0f6fe56ddc7451aa0dc5606b120f2/pytz-2018.4-py2.py3-none-any.whl Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.5.0.dev0) Collecting pyvcf<0.7.0,>=0.6.8 (from apache-beam==2.5.0.dev0) Collecting six<1.12,>=1.9 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/67/4b/141a581104b1f6397bfa78ac9d43d8ad29a7ca43ea90a2d863fe3056e86a/six-1.11.0-py2.py3-none-any.whl Collecting typing<3.7.0,>=3.6.0 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/0d/4d/4e5985d075d241d686a1663fa1f88b61d544658d08c1375c7c6aac32afc3/typing-3.6.4-py2-none-any.whl Collecting futures<4.0.0,>=3.1.1 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/2d/99/b2c4e9d5a30f6471e410a146232b4118e697fa3ffc06d6a65efde84debd0/futures-3.2.0-py2-none-any.whl Collecting google-apitools<=0.5.20,>=0.5.18 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/1d/0c/64f84f91643f775fdb64c6c10f4a4f0d827f8b0d98a2ba2b4bb9dc2f8646/google_apitools-0.5.20-py2-none-any.whl Collecting proto-google-cloud-datastore-v1<=0.90.4,>=0.90.0 (from apache-beam==2.5.0.dev0) Collecting googledatastore==7.0.1 (from apache-beam==2.5.0.dev0) Collecting google-cloud-pubsub==0.26.0 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/37/92/c74a643126d58505daec9addf872dfaffea3305981b90cc435f4b9213cdd/google_cloud_pubsub-0.26.0-py2.py3-none-any.whl Collecting proto-google-cloud-pubsub-v1==0.15.4 (from apache-beam==2.5.0.dev0) Collecting google-cloud-bigquery==0.25.0 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/76/67/6165c516ff6ceaa62eb61f11d8451e1b0acc4d3775e181630aba9652babb/google_cloud_bigquery-0.25.0-py2.py3-none-any.whl Collecting nose>=1.3.7 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl Collecting pyhamcrest<2.0,>=1.9 (from apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl Collecting enum34>=1.0.4 (from grpcio<2,>=1.8->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/49/df/50aa1999ab9bde74656c2919d9c0c085fd2b3775fd3eca826012bef76d8c/requests-2.18.4-py2.py3-none-any.whl Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) Collecting funcsigs>=1; python_version < "3.3" (from mock<3.0.0,>=1.0.1->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/69/cb/f5be453359271714c01b9bd06126eaf2e368f1fddfff30818754b5ac2328/funcsigs-1.0.2-py2.py3-none-any.whl Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/e1/ba/f95e3ec83f93919b1437028e989cf3fa5ff4f5cae4a1f62255f71deddb5b/pbr-4.0.2-py2.py3-none-any.whl Collecting rsa>=3.1.4 (from oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/e1/ae/baedc9cb175552e95f3395c43055a6a5e125ae4d48a1d7a924baca83e92e/rsa-3.4.2-py2.py3-none-any.whl Collecting pyasn1-modules>=0.0.5 (from oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/e9/51/bcd96bf6231d4b2cc5e023c511bee86637ba375c44a6f9d1b4b7ad1ce4b9/pyasn1_modules-0.2.1-py2.py3-none-any.whl Collecting pyasn1>=0.1.7 (from oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/ba/fe/02e3e2ee243966b143657fb8bd6bc97595841163b6d8c26820944acaec4d/pyasn1-0.4.2-py2.py3-none-any.whl Requirement already satisfied: setuptools in ./lib/python2.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.5.0.dev0) (39.0.1) Collecting fasteners>=0.14 (from google-apitools<=0.5.20,>=0.5.18->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/14/3a/096c7ad18e102d4f219f5dd15951f9728ca5092a3385d2e8f79a7c1e1017/fasteners-0.14.1-py2.py3-none-any.whl Collecting googleapis-common-protos<2.0dev,>=1.5.2 (from proto-google-cloud-datastore-v1<=0.90.4,>=0.90.0->apache-beam==2.5.0.dev0) Collecting google-cloud-core<0.26dev,>=0.25.0 (from google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/ef/dd/00e90bd1f6788f06ca5ea83a0ec8dd76350b38303bb8f09d2bf692eb1294/google_cloud_core-0.25.0-py2.py3-none-any.whl Collecting gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0 (from google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Collecting urllib3<1.23,>=1.21.1 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/63/cb/6965947c13a94236f6d4b8223e21beb4d576dc72e8130bd7880f600839b8/urllib3-1.22-py2.py3-none-any.whl Collecting idna<2.7,>=2.5 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/27/cc/6dd9a3869f15c2edfab863b992838277279ce92663d334df9ecf5106f5c6/idna-2.6-py2.py3-none-any.whl Collecting chardet<3.1.0,>=3.0.2 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl Collecting certifi>=2017.4.17 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0) <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/pip/_vendor/urllib3/util/ssl_.py>:339: SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name Indication) extension to TLS is not available on this platform. This may cause the server to present an incorrect TLS certificate, which can cause validation failures. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings SNIMissingWarning <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/pip/_vendor/urllib3/util/ssl_.py>:137: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings InsecurePlatformWarning Downloading https://files.pythonhosted.org/packages/7c/e6/92ad559b7192d846975fc916b65f667c7b8c3a32bea7372340bfe9a15fa5/certifi-2018.4.16-py2.py3-none-any.whl (150kB) Collecting monotonic>=0.1 (from fasteners>=0.14->google-apitools<=0.5.20,>=0.5.18->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/9f/ff/68860dec8b90424028ca54df8b9fc8cd1c47ef58710bd5e591ec8c619243/monotonic-1.4-py2.py3-none-any.whl Collecting google-auth<2.0.0dev,>=0.4.0 (from google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/e0/18/e282ecb28c7eb0ef5ea9a69fb3290658a9ab37bb69647f6ba7f0f78987c4/google_auth-1.4.1-py2.py3-none-any.whl Collecting google-auth-httplib2 (from google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/33/49/c814d6d438b823441552198f096fcd0377fd6c88714dbed34f1d3c8c4389/google_auth_httplib2-0.0.3-py2.py3-none-any.whl Collecting google-gax<0.16dev,>=0.15.7 (from gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/4f/b4/ff312fa42f91535c67567c1d08e972db0e7c548e9a63c6f3bcc5213b32fc/google_gax-0.15.16-py2.py3-none-any.whl Collecting grpc-google-iam-v1<0.12dev,>=0.11.1 (from gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Collecting cachetools>=2.0.0 (from google-auth<2.0.0dev,>=0.4.0->google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Using cached https://files.pythonhosted.org/packages/ac/e8/5492fd5ada0b05a1bc485bcb634b559acdec59383eef5c4203b5e22be296/cachetools-2.0.1-py2.py3-none-any.whl Collecting future<0.17dev,>=0.16.0 (from google-gax<0.16dev,>=0.15.7->gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) Collecting ply==3.8 (from google-gax<0.16dev,>=0.15.7->gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0) gapic-google-cloud-pubsub-v1 0.15.4 has requirement oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is incompatible. googledatastore 7.0.1 has requirement oauth2client<4.0.0,>=2.0.1, but you'll have oauth2client 4.1.2 which is incompatible. proto-google-cloud-pubsub-v1 0.15.4 has requirement oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is incompatible. proto-google-cloud-datastore-v1 0.90.4 has requirement oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is incompatible. Installing collected packages: avro, crcmod, dill, futures, six, protobuf, enum34, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, funcsigs, pbr, mock, pyasn1, rsa, pyasn1-modules, oauth2client, pytz, pyyaml, pyvcf, typing, monotonic, fasteners, google-apitools, googleapis-common-protos, proto-google-cloud-datastore-v1, googledatastore, cachetools, google-auth, google-auth-httplib2, google-cloud-core, future, ply, google-gax, grpc-google-iam-v1, proto-google-cloud-pubsub-v1, gapic-google-cloud-pubsub-v1, google-cloud-pubsub, google-cloud-bigquery, nose, pyhamcrest, apache-beam Running setup.py develop for apache-beam Successfully installed apache-beam avro-1.8.2 cachetools-2.0.1 certifi-2018.4.16 chardet-3.0.4 crcmod-1.7 dill-0.2.6 docopt-0.6.2 enum34-1.1.6 fasteners-0.14.1 funcsigs-1.0.2 future-0.16.0 futures-3.2.0 gapic-google-cloud-pubsub-v1-0.15.4 google-apitools-0.5.20 google-auth-1.4.1 google-auth-httplib2-0.0.3 google-cloud-bigquery-0.25.0 google-cloud-core-0.25.0 google-cloud-pubsub-0.26.0 google-gax-0.15.16 googleapis-common-protos-1.5.3 googledatastore-7.0.1 grpc-google-iam-v1-0.11.4 grpcio-1.11.0 hdfs-2.1.0 httplib2-0.9.2 idna-2.6 mock-2.0.0 monotonic-1.4 nose-1.3.7 oauth2client-4.1.2 pbr-4.0.2 ply-3.8 proto-google-cloud-datastore-v1-0.90.4 proto-google-cloud-pubsub-v1-0.15.4 protobuf-3.5.2.post1 pyasn1-0.4.2 pyasn1-modules-0.2.1 pyhamcrest-1.9.0 pytz-2018.4 pyvcf-0.6.8 pyyaml-3.12 requests-2.18.4 rsa-3.4.2 six-1.11.0 typing-3.6.4 urllib3-1.22 # Where to store integration test outputs. GCS_LOCATION=gs://temp-storage-for-end-to-end-tests PROJECT=apache-beam-testing # Create a tarball python setup.py sdist <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/setuptools/dist.py>:397: UserWarning: Normalizing '2.5.0.dev' to '2.5.0.dev0' normalized_version, running sdist <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py>:51: UserWarning: Installing grpcio-tools is recommended for development. warnings.warn('Installing grpcio-tools is recommended for development.') WARNING:root:Installing grpcio-tools into <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/.eggs/grpcio-wheels> Process Process-1: Traceback (most recent call last): File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run self._target(*self._args, **self._kwargs) File "<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py",> line 149, in _install_grpcio_tools_and_generate_proto_files shutil.rmtree(build_path) File "/usr/lib/python2.7/shutil.py", line 239, in rmtree onerror(os.listdir, path, sys.exc_info()) File "/usr/lib/python2.7/shutil.py", line 237, in rmtree names = os.listdir(path) OSError: [Errno 2] No such file or directory: '<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/.eggs/grpcio-wheels-build'> Traceback (most recent call last): File "setup.py", line 235, in <module> 'test': generate_protos_first(test), File "<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/setuptools/__init__.py",> line 129, in setup return distutils.core.setup(**attrs) File "/usr/lib/python2.7/distutils/core.py", line 151, in setup dist.run_commands() File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands self.run_command(cmd) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setup.py", line 141, in run gen_protos.generate_proto_files() File "<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py",> line 97, in generate_proto_files raise ValueError("Proto generation failed (see log for details).") ValueError: Proto generation failed (see log for details). Build step 'Execute shell' marked build as failure Not sending mail to unregistered user axelm...@gmail.com Not sending mail to unregistered user eh...@google.com Not sending mail to unregistered user apill...@google.com Not sending mail to unregistered user git...@alasdairhodge.co.uk Not sending mail to unregistered user alan.myrv...@comcast.net Not sending mail to unregistered user amyrv...@google.com Not sending mail to unregistered user ekirpic...@gmail.com Not sending mail to unregistered user sweg...@google.com Not sending mail to unregistered user sid...@google.com