Simone Pelosi has proposed merging
~pelpsi/launchpad:unembargo-bug-build-info-missing into launchpad:master.
Commit message:
Unembargo build info
Added buildinfo to unembargoed files.
Added custom upload files to unembargoed files.
Added a new endpoint buildMetadataFileUrls to retrieve
changes file, build_info and log without accessing directly the build.
LP: #2052796
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #2052796 in Launchpad itself: "MIssing buildinfo file for Jammy
grub2-unsigned 2.06-2ubuntu14.4"
https://bugs.launchpad.net/launchpad/+bug/2052796
For more details, see:
https://code.launchpad.net/~pelpsi/launchpad/+git/launchpad/+merge/464705
--
Your team Launchpad code reviewers is requested to review the proposed merge of
~pelpsi/launchpad:unembargo-bug-build-info-missing into launchpad:master.
diff --git a/lib/lp/soyuz/browser/tests/test_publishing_webservice.py b/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
index 2aee350..b167e61 100644
--- a/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
+++ b/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
@@ -15,9 +15,14 @@ from lp.services.librarian.browser import ProxiedLibraryFileAlias
from lp.services.webapp.interfaces import OAuthPermission
from lp.soyuz.adapters.proxiedsourcefiles import ProxiedSourceLibraryFileAlias
from lp.soyuz.enums import BinaryPackageFormat
-from lp.soyuz.interfaces.publishing import IPublishingSet
+from lp.soyuz.interfaces.publishing import (
+ IPublishingSet,
+ PackagePublishingStatus,
+)
+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
from lp.testing import (
TestCaseWithFactory,
+ admin_logged_in,
api_url,
login_person,
person_logged_in,
@@ -167,14 +172,47 @@ class SourcePackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
class BinaryPackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
layer = LaunchpadFunctionalLayer
+ def setUp(self):
+ super().setUp()
+ self.processor = self.factory.makeProcessor(supports_virtualized=True)
+ self.distroseries = self.factory.makeDistroSeries()
+ self.das = self.factory.makeDistroArchSeries(
+ distroseries=self.distroseries, processor=self.processor
+ )
+ with admin_logged_in():
+ self.publisher = SoyuzTestPublisher()
+ self.publisher.prepareBreezyAutotest()
+ self.distroseries.nominatedarchindep = self.das
+ self.publisher.addFakeChroots(distroseries=self.distroseries)
+ self.builder = self.factory.makeBuilder(
+ processors=[self.processor]
+ )
+
def make_bpph_for(self, person):
+ spn = self.factory.getUniqueString()
+ version = "%s.1" % self.factory.getUniqueInteger()
with person_logged_in(person):
- bpr = self.factory.makeBinaryPackageRelease()
- self.factory.makeBinaryPackageFile(binarypackagerelease=bpr)
- bpph = self.factory.makeBinaryPackagePublishingHistory(
- binarypackagerelease=bpr
+ spph = self.publisher.getPubSource(
+ sourcename=spn,
+ version=version,
+ distroseries=self.distroseries,
+ status=PackagePublishingStatus.PUBLISHED,
+ )
+ with admin_logged_in():
+ binary = self.publisher.getPubBinaries(
+ binaryname=spn,
+ distroseries=self.distroseries,
+ pub_source=spph,
+ version=version,
+ builder=self.builder,
+ )
+ binary[0].build.addBuildInfo(
+ self.factory.makeLibraryFileAlias(
+ filename="build_info.info", restricted=True
+ )
)
- return bpph, api_url(bpph)
+
+ return binary[0], api_url(binary[0])
def test_binaryFileUrls(self):
person = self.factory.makePerson()
@@ -239,6 +277,71 @@ class BinaryPackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
]
self.assertContentEqual(expected_info, info)
+ def test_buildMetadataFileUrls(self):
+ person = self.factory.makePerson()
+ webservice = webservice_for_person(
+ person, permission=OAuthPermission.READ_PUBLIC
+ )
+ bpph, url = self.make_bpph_for(person)
+
+ response = webservice.named_get(
+ url, "buildMetadataFileUrls", api_version="devel"
+ )
+
+ self.assertEqual(200, response.status)
+ urls = response.jsonBody()
+
+ with person_logged_in(person):
+ upload_changesfile = (
+ bpph.binarypackagerelease.build.upload_changesfile
+ )
+ log = bpph.binarypackagerelease.build.log
+ build_info = bpph.binarypackagerelease.build.buildinfo
+
+ expected_urls = [
+ ProxiedLibraryFileAlias(
+ upload_changesfile, bpph.archive
+ ).http_url,
+ ProxiedLibraryFileAlias(log, bpph.archive).http_url,
+ ProxiedLibraryFileAlias(build_info, bpph.archive).http_url,
+ ]
+ self.assertEqual(expected_urls, urls)
+
+ def test_buildMetadataFileUrls_include_meta(self):
+ person = self.factory.makePerson()
+ webservice = webservice_for_person(
+ person, permission=OAuthPermission.READ_PUBLIC
+ )
+ bpph, url = self.make_bpph_for(person)
+
+ response = webservice.named_get(
+ url,
+ "buildMetadataFileUrls",
+ include_meta=True,
+ api_version="devel",
+ )
+
+ self.assertEqual(200, response.status)
+ info = response.jsonBody()
+ with person_logged_in(person):
+ files = []
+ files.append(bpph.binarypackagerelease.build.upload_changesfile)
+ files.append(bpph.binarypackagerelease.build.log)
+ files.append(bpph.binarypackagerelease.build.buildinfo)
+
+ expected_info = [
+ {
+ "url": ProxiedLibraryFileAlias(
+ file, bpph.archive
+ ).http_url,
+ "size": file.content.filesize,
+ "sha1": file.content.sha1,
+ "sha256": file.content.sha256,
+ }
+ for file in files
+ ]
+ self.assertEqual(expected_info, info)
+
def test_ci_build(self):
person = self.factory.makePerson()
webservice = webservice_for_person(
diff --git a/lib/lp/soyuz/interfaces/publishing.py b/lib/lp/soyuz/interfaces/publishing.py
index a481aa3..c426ec7 100644
--- a/lib/lp/soyuz/interfaces/publishing.py
+++ b/lib/lp/soyuz/interfaces/publishing.py
@@ -1041,6 +1041,20 @@ class IBinaryPackagePublishingHistoryPublic(IPublishingView):
:return: A collection of URLs for this binary.
"""
+ @export_read_operation()
+ @operation_parameters(
+ include_meta=Bool(title=_("Include Metadata"), required=False)
+ )
+ @operation_for_version("devel")
+ def buildMetadataFileUrls(include_meta=False):
+ """URLs for this build metadata files.
+ The result includes changes file, build_info and log.
+
+ :param include_meta: Return a list of dicts with keys url, size,
+ sha1, and sha256 for each URL instead of a simple list.
+ :return: A collection of URLs for these metadata files.
+ """
+
class IBinaryPackagePublishingHistoryEdit(IPublishingEdit):
"""A writeable binary package publishing record."""
diff --git a/lib/lp/soyuz/model/packagecopyjob.py b/lib/lp/soyuz/model/packagecopyjob.py
index 2776af2..d30f3af 100644
--- a/lib/lp/soyuz/model/packagecopyjob.py
+++ b/lib/lp/soyuz/model/packagecopyjob.py
@@ -62,7 +62,7 @@ from lp.soyuz.interfaces.packagecopyjob import (
)
from lp.soyuz.interfaces.packagediff import PackageDiffAlreadyRequested
from lp.soyuz.interfaces.publishing import ISourcePackagePublishingHistory
-from lp.soyuz.interfaces.queue import IPackageUploadSet
+from lp.soyuz.interfaces.queue import IPackageUploadCustom, IPackageUploadSet
from lp.soyuz.interfaces.section import ISectionSet
from lp.soyuz.model.archive import Archive
from lp.soyuz.scripts.packagecopier import do_copy
@@ -854,7 +854,10 @@ class PlainPackageCopyJob(PackageCopyJobDerived):
"Packages copied to %s:" % self.target_archive.displayname
)
for copy in copied_publications:
- self.logger.debug(copy.displayname)
+ if IPackageUploadCustom.providedBy(copy):
+ self.logger.debug(copy.libraryfilealias.filename)
+ else:
+ self.logger.debug(copy.displayname)
def abort(self):
"""Abort work."""
diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py
index 78749c0..44543e0 100644
--- a/lib/lp/soyuz/model/publishing.py
+++ b/lib/lp/soyuz/model/publishing.py
@@ -1384,6 +1384,27 @@ class BinaryPackagePublishingHistory(StormBase, ArchivePublisherBase):
]
return binary_urls
+ def buildMetadataFileUrls(self, include_meta=False):
+ """See `IBinaryPackagePublishingHistory`."""
+ files = []
+ if self.build.upload_changesfile:
+ files.append(self.build.upload_changesfile)
+ if self.build.log:
+ files.append(self.build.log)
+ if self.build.buildinfo:
+ files.append(self.build.buildinfo)
+ file_urls = proxied_urls([file for file in files], self.archive)
+ if include_meta:
+ meta = [
+ (file.content.filesize, file.content.sha1, file.content.sha256)
+ for file in files
+ ]
+ return [
+ dict(url=url, size=size, sha1=sha1, sha256=sha256)
+ for url, (size, sha1, sha256) in zip(file_urls, meta)
+ ]
+ return file_urls
+
def expand_binary_requests(distroseries, binaries):
"""Architecture-expand a dict of binary publication requests.
diff --git a/lib/lp/soyuz/model/queue.py b/lib/lp/soyuz/model/queue.py
index 7c7fda2..ac633e1 100644
--- a/lib/lp/soyuz/model/queue.py
+++ b/lib/lp/soyuz/model/queue.py
@@ -635,10 +635,13 @@ class PackageUpload(StormBase):
# XXX: Only tests are not passing user here. We should adjust the
# tests and always create the log entries after
if user is not None:
+ print("HERE")
self._addLog(user, PackageUploadStatus.ACCEPTED, None)
if self.package_copy_job is None:
+ print("HERE1")
self._acceptNonSyncFromQueue()
else:
+ print("HERE2")
self._acceptSyncFromQueue()
def rejectFromQueue(self, user, comment=None):
diff --git a/lib/lp/soyuz/scripts/packagecopier.py b/lib/lp/soyuz/scripts/packagecopier.py
index f377107..cd3c624 100644
--- a/lib/lp/soyuz/scripts/packagecopier.py
+++ b/lib/lp/soyuz/scripts/packagecopier.py
@@ -73,6 +73,8 @@ def update_files_privacy(pub_record):
package_files.append((sourcepackagerelease, "changelog"))
elif IBinaryPackagePublishingHistory.providedBy(pub_record):
archive = pub_record.archive
+ print("IS PRIVATE?")
+ print(pub_record.archive.private)
# Unrestrict the binary files if necessary.
binarypackagerelease = pub_record.binarypackagerelease
package_files.extend(
@@ -87,6 +89,8 @@ def update_files_privacy(pub_record):
package_files.append((package_upload, "changesfile"))
# Unrestrict the buildlog file as necessary.
package_files.append((build, "log"))
+ # Unrestrict the buildinfo file as necessary.
+ package_files.append((build, "buildinfo"))
elif IPackageUploadCustom.providedBy(pub_record):
# Unrestrict the custom files included
package_files.append((pub_record, "libraryfilealias"))
@@ -911,7 +915,7 @@ def _do_direct_copy(
)
for custom in custom_files:
if custom_copier.isCopyable(custom):
- custom_copier.copyUpload(custom)
+ copies.append(custom_copier.copyUpload(custom))
# Always ensure the needed builds exist in the copy destination
# after copying the binaries.
diff --git a/lib/lp/soyuz/tests/test_packagecopyjob.py b/lib/lp/soyuz/tests/test_packagecopyjob.py
index 26fdd08..be893fb 100644
--- a/lib/lp/soyuz/tests/test_packagecopyjob.py
+++ b/lib/lp/soyuz/tests/test_packagecopyjob.py
@@ -31,6 +31,7 @@ from lp.services.features.testing import FeatureFixture
from lp.services.job.interfaces.job import JobStatus
from lp.services.job.runner import JobRunner
from lp.services.job.tests import block_on_job, pop_remote_notifications
+from lp.services.macaroons.testing import MacaroonTestMixin
from lp.services.mail.sendmail import format_address_for_person
from lp.soyuz.adapters.overrides import SourceOverride
from lp.soyuz.enums import (
@@ -195,7 +196,9 @@ class LocalTestHelper:
JobRunner([job]).runAll()
-class PlainPackageCopyJobTests(TestCaseWithFactory, LocalTestHelper):
+class PlainPackageCopyJobTests(
+ TestCaseWithFactory, LocalTestHelper, MacaroonTestMixin
+):
"""Test case for PlainPackageCopyJob."""
layer = LaunchpadZopelessLayer
@@ -1733,6 +1736,114 @@ class PlainPackageCopyJobTests(TestCaseWithFactory, LocalTestHelper):
self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
self.assertEqual(1, webhook.deliveries.count())
+ def test_copying_unembargoes_build_files(self):
+ # Inaccessible private builds aren't linked in builders'
+ # current_build fields.
+ # Copyable custom upload files are queued for republication when
+ # they are copied.
+ # Custom upload files should be unembargoed as well.
+
+ owner = self.factory.makePerson()
+
+ source_archive = self.factory.makeArchive(
+ name="sourcearchive",
+ owner=owner,
+ distribution=self.distroseries.distribution,
+ private=True,
+ )
+ target_archive = self.factory.makeArchive(
+ name="targetarchive",
+ owner=owner,
+ distribution=self.distroseries.distribution,
+ purpose=ArchivePurpose.PRIMARY,
+ )
+
+ self.distroseries.status = SeriesStatus.CURRENT
+ spph = self.publisher.getPubSource(
+ pocket=PackagePublishingPocket.SECURITY, archive=source_archive
+ )
+ self.publisher.getPubBinaries(
+ pocket=PackagePublishingPocket.SECURITY,
+ pub_source=spph,
+ archive=source_archive,
+ )
+ [build] = spph.getBuilds()
+ build.addBuildInfo(
+ self.factory.makeLibraryFileAlias(
+ filename="build_info.info", restricted=True
+ )
+ )
+ build.package_upload.addCustom(
+ self.factory.makeLibraryFileAlias(
+ filename="test.deb", restricted=True
+ ),
+ PackageUploadCustomFormat.DEBIAN_INSTALLER,
+ )
+ # Make the new librarian file available.
+ self.layer.txn.commit()
+ # Create the copy job.
+ with person_logged_in(owner):
+ target_archive.newPocketUploader(
+ owner, PackagePublishingPocket.SECURITY
+ )
+ job = self.createCopyJobForSPPH(
+ spph,
+ source_archive,
+ target_archive,
+ requester=owner,
+ target_pocket=PackagePublishingPocket.SECURITY,
+ include_binaries=True,
+ unembargo=True,
+ )
+
+ # Start, accept, and run the job.
+ self.runJob(job)
+ self.assertEqual(JobStatus.SUSPENDED, job.status)
+ switch_dbuser("launchpad_main")
+ pu = (
+ getUtility(IPackageUploadSet)
+ .getByPackageCopyJobIDs([removeSecurityProxy(job).context.id])
+ .one()
+ )
+ pu.acceptFromQueue()
+ self.assertEqual(PackageUploadStatus.ACCEPTED, pu.status)
+ self.runJob(job)
+ self.assertEqual(PackageUploadStatus.DONE, pu.status)
+
+ uploads = list(
+ self.distroseries.getPackageUploads(
+ status=PackageUploadStatus.ACCEPTED,
+ archive=target_archive,
+ pocket=PackagePublishingPocket.SECURITY,
+ )
+ )
+
+ self.assertEqual(1, len(uploads))
+ upload = uploads[0]
+ self.assertEqual(
+ PackageUploadCustomFormat.DEBIAN_INSTALLER,
+ upload.customfiles[0].customformat,
+ )
+ # The upload is targeted to the right publishing context.
+ self.assertEqual(target_archive, upload.archive)
+ self.assertEqual(self.distroseries, upload.distroseries)
+ self.assertEqual(PackagePublishingPocket.SECURITY, upload.pocket)
+
+ self.assertEqual(
+ "test.deb", upload.customfiles[0].libraryfilealias.filename
+ )
+ self.assertEqual(
+ False, upload.customfiles[0].libraryfilealias.restricted
+ )
+ copied_binaries = target_archive.getAllPublishedBinaries()
+ for copied_binary in copied_binaries:
+ for binary_file in copied_binary.binarypackagerelease.files:
+ self.assertFalse(binary_file.libraryfile.restricted)
+ copied_build = copied_binary.binarypackagerelease.build
+ self.assertFalse(copied_build.log.restricted)
+ self.assertFalse(copied_build.buildinfo.restricted)
+ self.assertFalse(copied_build.upload_changesfile.restricted)
+
def test_copying_unembargoes_files(self):
# The unembargo flag causes the job to unrestrict files.
self.distroseries.status = SeriesStatus.CURRENT
diff --git a/lib/lp/soyuz/tests/test_publishing_models.py b/lib/lp/soyuz/tests/test_publishing_models.py
index 755d797..e529d52 100644
--- a/lib/lp/soyuz/tests/test_publishing_models.py
+++ b/lib/lp/soyuz/tests/test_publishing_models.py
@@ -21,7 +21,8 @@ from lp.soyuz.interfaces.publishing import (
PackagePublishingStatus,
)
from lp.soyuz.tests.test_binarypackagebuild import BaseTestCaseWithThreeBuilds
-from lp.testing import TestCaseWithFactory, person_logged_in
+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
+from lp.testing import TestCaseWithFactory, admin_logged_in, person_logged_in
from lp.testing.layers import LaunchpadFunctionalLayer, LaunchpadZopelessLayer
@@ -242,6 +243,22 @@ class TestSourcePackagePublishingHistory(TestCaseWithFactory):
class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
layer = LaunchpadFunctionalLayer
+ def setUp(self):
+ super().setUp()
+ self.processor = self.factory.makeProcessor(supports_virtualized=True)
+ self.distroseries = self.factory.makeDistroSeries()
+ self.das = self.factory.makeDistroArchSeries(
+ distroseries=self.distroseries, processor=self.processor
+ )
+ with admin_logged_in():
+ self.publisher = SoyuzTestPublisher()
+ self.publisher.prepareBreezyAutotest()
+ self.distroseries.nominatedarchindep = self.das
+ self.publisher.addFakeChroots(distroseries=self.distroseries)
+ self.builder = self.factory.makeBuilder(
+ processors=[self.processor]
+ )
+
def getURLsForBPPH(self, bpph, include_meta=False):
bpr = bpph.binarypackagerelease
archive = bpph.archive
@@ -278,11 +295,35 @@ class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
binarypackagerelease=bpr, archive=archive
)
+ def makeBPPHWithMetadata(self):
+ spn = self.factory.getUniqueString()
+ version = "%s.1" % self.factory.getUniqueInteger()
+ with admin_logged_in():
+ spph = self.publisher.getPubSource(
+ sourcename=spn,
+ version=version,
+ distroseries=self.distroseries,
+ status=PackagePublishingStatus.PUBLISHED,
+ )
+ binary = self.publisher.getPubBinaries(
+ binaryname=spn,
+ distroseries=self.distroseries,
+ pub_source=spph,
+ version=version,
+ builder=self.builder,
+ )
+ binary[0].build.addBuildInfo(
+ self.factory.makeLibraryFileAlias(
+ filename="build_info.info", restricted=True
+ )
+ )
+
+ return binary[0]
+
def test_binaryFileUrls_no_binaries(self):
bpph = self.makeBPPH(num_binaries=0)
urls = bpph.binaryFileUrls()
-
self.assertContentEqual([], urls)
def test_binaryFileUrls_one_binary(self):
@@ -324,6 +365,55 @@ class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
expected_urls_meta, bpph.binaryFileUrls(include_meta=True)
)
+ def test_buildMetadataFileUrls(self):
+ bpph = self.makeBPPHWithMetadata()
+
+ urls = bpph.buildMetadataFileUrls()
+ with admin_logged_in():
+ upload_changesfile = (
+ bpph.binarypackagerelease.build.upload_changesfile
+ )
+ log = bpph.binarypackagerelease.build.log
+ build_info = bpph.binarypackagerelease.build.buildinfo
+
+ expected_urls = [
+ ProxiedLibraryFileAlias(
+ upload_changesfile, bpph.archive
+ ).http_url,
+ ProxiedLibraryFileAlias(log, bpph.archive).http_url,
+ ProxiedLibraryFileAlias(build_info, bpph.archive).http_url,
+ ]
+ self.assertContentEqual(expected_urls, urls)
+
+ def test_buildMetadataFileUrls_include_meta(self):
+ bpph = self.makeBPPHWithMetadata()
+
+ urls = bpph.buildMetadataFileUrls(True)
+ with admin_logged_in():
+ files = []
+ files.append(bpph.binarypackagerelease.build.upload_changesfile)
+ files.append(bpph.binarypackagerelease.build.log)
+ files.append(bpph.binarypackagerelease.build.buildinfo)
+
+ expected_urls = [
+ {
+ "url": ProxiedLibraryFileAlias(
+ file, bpph.archive
+ ).http_url,
+ "size": file.content.filesize,
+ "sha1": file.content.sha1,
+ "sha256": file.content.sha256,
+ }
+ for file in files
+ ]
+ self.assertContentEqual(expected_urls, urls)
+
+ def test_buildMetadataFileUrls_no_files(self):
+ bpph = self.makeBPPH(num_binaries=0)
+
+ urls = bpph.buildMetadataFileUrls()
+ self.assertContentEqual([], urls)
+
def test_is_debug_false_for_deb(self):
bpph = self.factory.makeBinaryPackagePublishingHistory(
binpackageformat=BinaryPackageFormat.DEB
_______________________________________________
Mailing list: https://launchpad.net/~launchpad-reviewers
Post to : launchpad-reviewers@lists.launchpad.net
Unsubscribe : https://launchpad.net/~launchpad-reviewers
More help : https://help.launchpad.net/ListHelp
