The issue was with the pip version. --download was deprecated. I dont know
where this need to be mentioned / fixed.
running
pip install pip==9.0.3
solved the issue.
Thanks,
eila
On Wed, Aug 22, 2018 at 11:20 AM OrielResearch Eila Arich-Landkof <
e...@orielresearch.org> wrote:
> I tried a simple pipeline which is runner perfectly on local runner and
> the same issue on dataflow. see below. Is there anything at the environment
> that need to be updated that I am not aware of?
>
> Many thanks for any reference.
> Eila
>
> import apache_beam as beam
> options = PipelineOptions()
> google_cloud_options = options.view_as(GoogleCloudOptions)
> google_cloud_options.project = 'PROJECT-ID'
> google_cloud_options.job_name = 'try-debug'
> google_cloud_options.staging_location = '%s/staging' % BUCKET_URL
> #'gs://archs4/staging'
> google_cloud_options.temp_location = '%s/tmp' % BUCKET_URL #'gs://archs4/temp'
> options.view_as(StandardOptions).runner = 'DataflowRunner'
>
> p1 = beam.Pipeline(options=options)
>
> (p1 | 'read' >>
> beam.io.ReadFromText('gs://dataflow-samples/shakespeare/kinglear.txt')
> | 'write' >> beam.io.WriteToText('gs://bucket/test.txt', num_shards=1)
> )
>
> p1.run().wait_until_finish()
>
> will fire the following error:
>
> CalledProcessErrorTraceback (most recent call last)
> <ipython-input-17-b4be63f7802f> in <module>()
> 5 )
> 6
> ----> 7 p1.run().wait_until_finish()
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/pipeline.pyc
> in run(self, test_runner_api)
> 174 finally:
> 175 shutil.rmtree(tmpdir)
> --> 176 return self.runner.run(self)
> 177
> 178 def __enter__(self):
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/dataflow_runner.pyc
> in run(self, pipeline)
> 250 # Create the job
> 251 result = DataflowPipelineResult(
> --> 252 self.dataflow_client.create_job(self.job), self)
> 253
> 254 self._metrics = DataflowMetrics(self.dataflow_client, result,
> self.job)
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/utils/retry.pyc
> in wrapper(*args, **kwargs)
> 166 while True:
> 167 try:
> --> 168 return fun(*args, **kwargs)
> 169 except Exception as exn: # pylint: disable=broad-except
> 170 if not retry_filter(exn):
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/apiclient.pyc
> in create_job(self, job)
> 423 def create_job(self, job):
> 424 """Creates job description. May stage and/or submit for remote
> execution."""
> --> 425 self.create_job_description(job)
> 426
> 427 # Stage and submit the job when necessary
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/apiclient.pyc
> in create_job_description(self, job)
> 446 """Creates a job described by the workflow proto."""
> 447 resources = dependency.stage_job_resources(
> --> 448 job.options, file_copy=self._gcs_file_copy)
> 449 job.proto.environment = Environment(
> 450 packages=resources, options=job.options,
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
> in stage_job_resources(options, file_copy, build_setup_args, temp_dir,
> populate_requirements_cache)
> 377 else:
> 378 sdk_remote_location = setup_options.sdk_location
> --> 379 _stage_beam_sdk_tarball(sdk_remote_location, staged_path,
> temp_dir)
> 380 resources.append(names.DATAFLOW_SDK_TARBALL_FILE)
> 381 else:
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
> in _stage_beam_sdk_tarball(sdk_remote_location, staged_path, temp_dir)
> 462 elif sdk_remote_location == 'pypi':
> 463 logging.info('Staging the SDK tarball from PyPI to %s',
> staged_path)
> --> 464 _dependency_file_copy(_download_pypi_sdk_package(temp_dir),
> staged_path)
> 465 else:
> 466 raise RuntimeError(
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
> in _download_pypi_sdk_package(temp_dir)
> 525 '--no-binary', ':all:', '--no-deps']
> 526 logging.info('Executing command: %s', cmd_args)
> --> 527 processes.check_call(cmd_args)
> 528 zip_expected = os.path.join(
> 529 temp_dir, '%s-%s.zip' % (package_name, version))
>
> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/utils/processes.pyc
> in check_call(*args, **kwargs)
> 42 if force_shell:
> 43 kwargs['shell'] = True
> ---> 44 return subprocess.check_call(*args, **kwargs)
> 45
> 46
>
> /usr/local/envs/py2env/lib/python2.7/subprocess.pyc in check_call(*popenargs,
> **kwargs)
> 188 if cmd is None:
> 189 cmd = popenargs[0]
> --> 190 raise CalledProcessError(retcode, cmd)
> 191 return 0
> 192
>
> CalledProcessError: Command '['/usr/local/envs/py2env/bin/python', '-m',
> 'pip', 'install', '--download', '/tmp/tmpyyiizo',
> 'google-cloud-dataflow==2.0.0', '--no-binary', ':all:', '--no-deps']'
> returned non-zero exit status 2
>
>
>
> On Wed, Aug 22, 2018 at 10:39 AM OrielResearch Eila Arich-Landkof <
> e...@orielresearch.org> wrote:
>
>> Hello all,
>>
>> I am running a pipeline that used to be executed on dataflow with no
>> issues. I am using the datalab environment. See below the error. To my
>> understanding happening before the pipeline code is being is being executed.
>> Any idea what went wrong?
>>
>> Thanks,
>> Eila
>>
>>
>> Executing the pipeline:
>>
>> *p.run().wait_until_finish()*
>>
>> The following error is being fired:
>>
>> INFO:root:Executing command: ['/usr/local/envs/py2env/bin/python',
>> 'setup.py', 'sdist', '--dist-dir', '/tmp/tmp_B0gnK']
>> INFO:root:Starting GCS upload to
>> gs://archs4/staging/label-archs4-annotation-15.1534948236.075799/workflow.tar.gz...
>> INFO:oauth2client.client:Attempting refresh to obtain initial access_token
>> INFO:root:Completed GCS upload to
>> gs://archs4/staging/label-archs4-annotation-15.1534948236.075799/workflow.tar.gz
>> INFO:root:Staging the SDK tarball from PyPI to
>> gs://archs4/staging/label-archs4-annotation-15.1534948236.075799/dataflow_python_sdk.tar
>> INFO:root:Executing command: ['/usr/local/envs/py2env/bin/python', '-m',
>> 'pip', 'install', '--download', '/tmp/tmp_B0gnK',
>> 'google-cloud-dataflow==2.0.0', '--no-binary', ':all:', '--no-deps']
>>
>> CalledProcessErrorTraceback (most recent call
>> last)<ipython-input-27-1e5aeb8b7d9b> in <module>()----> 1
>> p.run().wait_until_finish()
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/pipeline.pyc
>> in run(self, test_runner_api) 174 finally: 175
>> shutil.rmtree(tmpdir)--> 176 return self.runner.run(self) 177 178
>> def __enter__(self):
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/dataflow_runner.pyc
>> in run(self, pipeline) 250 # Create the job 251 result =
>> DataflowPipelineResult(--> 252
>> self.dataflow_client.create_job(self.job), self) 253 254
>> self._metrics = DataflowMetrics(self.dataflow_client, result, self.job)
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/utils/retry.pyc
>> in wrapper(*args, **kwargs) 166 while True: 167 try:-->
>> 168 return fun(*args, **kwargs) 169 except Exception as
>> exn: # pylint: disable=broad-except 170 if not
>> retry_filter(exn):
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/apiclient.pyc
>> in create_job(self, job) 423 def create_job(self, job): 424
>> """Creates job description. May stage and/or submit for remote
>> execution."""--> 425 self.create_job_description(job) 426 427
>> # Stage and submit the job when necessary
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/apiclient.pyc
>> in create_job_description(self, job) 446 """Creates a job described
>> by the workflow proto.""" 447 resources =
>> dependency.stage_job_resources(--> 448 job.options,
>> file_copy=self._gcs_file_copy) 449 job.proto.environment =
>> Environment( 450 packages=resources, options=job.options,
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
>> in stage_job_resources(options, file_copy, build_setup_args, temp_dir,
>> populate_requirements_cache) 377 else: 378
>> sdk_remote_location = setup_options.sdk_location--> 379
>> _stage_beam_sdk_tarball(sdk_remote_location, staged_path, temp_dir) 380
>> resources.append(names.DATAFLOW_SDK_TARBALL_FILE) 381 else:
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
>> in _stage_beam_sdk_tarball(sdk_remote_location, staged_path, temp_dir)
>> 462 elif sdk_remote_location == 'pypi': 463 logging.info('Staging
>> the SDK tarball from PyPI to %s', staged_path)--> 464
>> _dependency_file_copy(_download_pypi_sdk_package(temp_dir), staged_path)
>> 465 else: 466 raise RuntimeError(
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/runners/dataflow/internal/dependency.pyc
>> in _download_pypi_sdk_package(temp_dir) 525 '--no-binary',
>> ':all:', '--no-deps'] 526 logging.info('Executing command: %s',
>> cmd_args)--> 527 processes.check_call(cmd_args) 528 zip_expected =
>> os.path.join( 529 temp_dir, '%s-%s.zip' % (package_name, version))
>> /usr/local/envs/py2env/lib/python2.7/site-packages/apache_beam/utils/processes.pyc
>> in check_call(*args, **kwargs) 42 if force_shell: 43
>> kwargs['shell'] = True---> 44 return subprocess.check_call(*args,
>> **kwargs) 45 46
>> /usr/local/envs/py2env/lib/python2.7/subprocess.pyc in
>> check_call(*popenargs, **kwargs) 188 if cmd is None: 189
>> cmd = popenargs[0]--> 190 raise CalledProcessError(retcode,
>> cmd) 191 return 0 192
>> CalledProcessError: Command '['/usr/local/envs/py2env/bin/python', '-m',
>> 'pip', 'install', '--download', '/tmp/tmp_B0gnK',
>> 'google-cloud-dataflow==2.0.0', '--no-binary', ':all:', '--no-deps']'
>> returned non-zero exit status 2
>>
>>
>>
>> --
>> Eila
>> www.orielresearch.org
>> https://www.meetu <https://www.meetup.com/Deep-Learning-In-Production/>
>> p.co <https://www.meetup.com/Deep-Learning-In-Production/>
>> m/Deep-Learning-In-Production/
>> <https://www.meetup.com/Deep-Learning-In-Production/>
>>
>>
>>
>
> --
> Eila
> www.orielresearch.org
> https://www.meetu <https://www.meetup.com/Deep-Learning-In-Production/>
> p.co <https://www.meetup.com/Deep-Learning-In-Production/>
> m/Deep-Learning-In-Production/
> <https://www.meetup.com/Deep-Learning-In-Production/>
>
>
>
--
Eila
www.orielresearch.org
https://www.meetu <https://www.meetup.com/Deep-Learning-In-Production/>p.co
<https://www.meetup.com/Deep-Learning-In-Production/>
m/Deep-Learning-In-Production/
<https://www.meetup.com/Deep-Learning-In-Production/>
