This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6e8f0e4d32 ARROW-17233: [Packaging][Linux] Update artifact patterns
(#13740)
6e8f0e4d32 is described below
commit 6e8f0e4d327180375dda53287a5a600ba139ce3d
Author: Sutou Kouhei <[email protected]>
AuthorDate: Thu Aug 18 09:23:06 2022 +0900
ARROW-17233: [Packaging][Linux] Update artifact patterns (#13740)
Add `--validate` option to `archery crossbow status`.
If `--validate` is specified and there are any missing artifacts, `archery
crossbow status --validate` is existed
with non-zero exit code. We can use it for CI to detect missing artifacts.
We can't use `@github-actions crossbow submit` for this change because this
isn't merged into the master branch
yet. See
https://github.com/ursacomputing/crossbow/branches/all?query=build-674 that is
submitted `nightly-packages`
manually.
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
dev/archery/archery/crossbow/cli.py | 19 ++++++++++--
dev/archery/archery/crossbow/core.py | 18 +++++++++---
dev/tasks/macros.jinja | 56 +++++++++++++++++++++++++++---------
dev/tasks/tasks.yml | 10 ++++---
4 files changed, 79 insertions(+), 24 deletions(-)
diff --git a/dev/archery/archery/crossbow/cli.py
b/dev/archery/archery/crossbow/cli.py
index 8bdc490958..d34b22bd9e 100644
--- a/dev/archery/archery/crossbow/cli.py
+++ b/dev/archery/archery/crossbow/cli.py
@@ -17,6 +17,7 @@
from pathlib import Path
import time
+import sys
import click
@@ -273,16 +274,30 @@ def render(obj, task, config_path, arrow_version,
arrow_remote, arrow_branch,
help='Fetch references (branches and tags) from the remote')
@click.option('--task-filter', '-f', 'task_filters', multiple=True,
help='Glob pattern for filtering relevant tasks')
[email protected]('--validate/--no-validate', default=False,
+ help='Return non-zero exit code '
+ 'if there is any non-success task')
@click.pass_obj
-def status(obj, job_name, fetch, task_filters):
+def status(obj, job_name, fetch, task_filters, validate):
output = obj['output']
queue = obj['queue']
if fetch:
queue.fetch()
job = queue.get(job_name)
+ success = True
+
+ def asset_callback(task_name, task, asset):
+ nonlocal success
+ if task.status().combined_state in {'error', 'failure'}:
+ success = False
+ if asset is None:
+ success = False
+
report = ConsoleReport(job, task_filters=task_filters)
- report.show(output)
+ report.show(output, asset_callback=asset_callback)
+ if validate and not success:
+ sys.exit(1)
@crossbow.command()
diff --git a/dev/archery/archery/crossbow/core.py
b/dev/archery/archery/crossbow/core.py
index 49f915ec82..75b489f048 100644
--- a/dev/archery/archery/crossbow/core.py
+++ b/dev/archery/archery/crossbow/core.py
@@ -675,6 +675,7 @@ class Queue(Repo):
params = {
**job.params,
"arrow": job.target,
+ "job": job,
"queue_remote_url": self.remote_url
}
files = task.render_files(job.template_searchpath, params=params)
@@ -790,7 +791,7 @@ class Task(Serializable):
submitting the job to a queue.
"""
- def __init__(self, ci, template, artifacts=None, params=None):
+ def __init__(self, name, ci, template, artifacts=None, params=None):
assert ci in {
'circle',
'travis',
@@ -799,6 +800,7 @@ class Task(Serializable):
'github',
'drone',
}
+ self.name = name
self.ci = ci
self.template = template
self.artifacts = artifacts or []
@@ -1011,6 +1013,7 @@ class Job(Serializable):
params = {
**self.params,
"arrow": self.target,
+ "job": self,
**(params or {})
}
for task_name, task in self.tasks.items():
@@ -1081,9 +1084,10 @@ class Job(Serializable):
'no_rc_version': target.no_rc_version,
'no_rc_semver_version': target.no_rc_semver_version}
for task_name, task in task_definitions.items():
+ task = task.copy()
artifacts = task.pop('artifacts', None) or [] # because of yaml
artifacts = [fn.format(**versions) for fn in artifacts]
- tasks[task_name] = Task(artifacts=artifacts, **task)
+ tasks[task_name] = Task(task_name, artifacts=artifacts, **task)
return cls(target=target, tasks=tasks, params=params,
template_searchpath=config.template_searchpath)
@@ -1219,7 +1223,7 @@ class Config(dict):
# validate that the tasks are constructible
for task_name, task in self['tasks'].items():
try:
- Task(**task)
+ Task(task_name, **task)
except Exception as e:
raise CrossbowError(
'Unable to construct a task object from the '
@@ -1236,13 +1240,19 @@ class Config(dict):
version='1.0.0dev123',
email='[email protected]'
)
+ job = Job.from_config(config=self,
+ target=target,
+ tasks=self['tasks'],
+ groups=self['groups'],
+ params={})
for task_name, task in self['tasks'].items():
- task = Task(**task)
+ task = Task(task_name, **task)
files = task.render_files(
self.template_searchpath,
params=dict(
arrow=target,
+ job=job,
queue_remote_url='https://github.com/org/crossbow'
)
)
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index bbabad69a7..09a6696a37 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -69,25 +69,43 @@ on:
uses: actions/setup-python@v4
with:
python-version: 3.8
+ - name: Checkout Crossbow
+ uses: actions/checkout@v3
+ with:
+ path: crossbow
+ ref: {{ job.branch }}
- name: Setup Crossbow
shell: bash
- run: pip install -e arrow/dev/archery[crossbow-upload]
+ run: pip install -e arrow/dev/archery[crossbow]
- name: Upload artifacts
shell: bash
run: |
archery crossbow \
- --queue-path $(pwd) \
- --queue-remote {{ queue_remote_url }} \
- upload-artifacts \
- --sha {{ task.branch }} \
- --tag {{ task.tag }} \
- {% if pattern is string %}
- "{{ pattern }}"
- {% elif pattern is iterable %}
- {% for p in pattern %}
- "{{ p }}" {{ "\\" if not loop.last else "" }}
- {% endfor %}
- {% endif %}
+ --queue-path $(pwd)/crossbow \
+ --queue-remote {{ queue_remote_url }} \
+ upload-artifacts \
+ --sha {{ task.branch }} \
+ --tag {{ task.tag }} \
+ {% if pattern is string %}
+ "{{ pattern }}"
+ {% elif pattern is iterable %}
+ {% for p in pattern %}
+ "{{ p }}" {{ "\\" if not loop.last else "" }}
+ {% endfor %}
+ {% endif %}
+ env:
+ CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN ||
secrets.GITHUB_TOKEN }}' }}
+ - name: Verify uploaded artifacts
+ shell: bash
+ run: |
+ archery crossbow \
+ --queue-path $(pwd)/crossbow \
+ --queue-remote {{ queue_remote_url }} \
+ status \
+ --task-filter '{{ task.name }}' \
+ --no-fetch \
+ --validate \
+ {{ job.branch }}
env:
CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN ||
secrets.GITHUB_TOKEN }}' }}
{% endmacro %}
@@ -170,7 +188,7 @@ on:
{%- macro travis_upload_releases(pattern) -%}
- sudo -H pip3 install pygit2==1.0 cryptography==36
- - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload]
+ - sudo -H pip3 install -e arrow/dev/archery[crossbow]
- |
archery crossbow \
--queue-path $(pwd) \
@@ -185,6 +203,16 @@ on:
"{{ p }}" {{ "\\" if not loop.last else "" }}
{% endfor %}
{% endif %}
+ - git fetch origin {{ job.branch }}:remotes/origin/{{ job.branch }}
+ - |
+ archery crossbow \
+ --queue-path $(pwd) \
+ --queue-remote {{ queue_remote_url }} \
+ status \
+ --task-filter '{{ task.name }}' \
+ --no-fetch \
+ --validate \
+ {{ job.branch }}
{% endmacro %}
{%- macro travis_upload_gemfury(pattern) -%}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index a07c4d878c..45a396d6d8 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -627,12 +627,14 @@ tasks:
- libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
- libarrow-glib1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
- libarrow-glib1000_{no_rc_version}-1_[a-z0-9]+.deb
+ {% if target != "ubuntu-bionic" %}
- libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
- libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
- libarrow-python-flight1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
- libarrow-python-flight1000_{no_rc_version}-1_[a-z0-9]+.deb
- libarrow-python1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
- libarrow-python1000_{no_rc_version}-1_[a-z0-9]+.deb
+ {% endif %}
- libarrow1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
- libarrow1000_{no_rc_version}-1_[a-z0-9]+.deb
- libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
@@ -720,7 +722,7 @@ tasks:
- arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
{% endif %}
- arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- {% if not is_rhel7_based %}
+ {% if not is_rhel7_based and architecture == "amd64" %}
- arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
@@ -746,14 +748,14 @@ tasks:
- arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
{% endif %}
- arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- {% if target != "amazon-linux-2" %}
+ {% if not is_rhel7_based %}
- arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- {% if not is_rhel7_based %}
+ {% if architecture == "amd64" %}
- arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-
arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-
arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- -
arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
{% endif %}
+ -
arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
- arrow[0-9]+-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
{% endif %}
{% if architecture == "amd64" %}
