Jürgen Gmach has proposed merging ~jugmac00/launchpad:attach-build-output-to-revision-status-reports into launchpad:master.
Commit message: Attach output of completed lpcraft builds Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~jugmac00/launchpad/+git/launchpad/+merge/415782 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad:attach-build-output-to-revision-status-reports into launchpad:master.
diff --git a/lib/lp/archiveuploader/ciupload.py b/lib/lp/archiveuploader/ciupload.py new file mode 100644 index 0000000..9a8b9ef --- /dev/null +++ b/lib/lp/archiveuploader/ciupload.py @@ -0,0 +1,96 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Process a CI upload.""" + +__all__ = [ + "CIUpload", + ] + +import json +import os + +from zope.component import getUtility + +from lp.archiveuploader.utils import UploadError +from lp.buildmaster.enums import BuildStatus +from lp.code.enums import RevisionStatusResult +from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet + + +class CIUpload: + """An upload from a pipeline of CI jobs.""" + + def __init__(self, upload_path, logger): + """Create a `CIUpload`. + + :param upload_path: A directory containing files to upload. + :param logger: The logger to be used. + """ + self.upload_path = upload_path + self.logger = logger + + def process(self, build): + """Process this upload, loading it into the database.""" + self.logger.debug("Beginning processing.") + + jobs_path = os.path.join(self.upload_path, "jobs.json") + try: + with open(jobs_path) as jobs_file: + jobs = json.load(jobs_file) + except FileNotFoundError: + raise UploadError("Build did not run any jobs.") + + # collect all artifacts + artifacts = {} + # we assume first level directories are job directories + job_directories = ( + d.name for d in os.scandir(self.upload_path) if d.is_dir() + ) + for job_directory in job_directories: + artifacts[job_directory] = [] + for dirpath, _, filenames in os.walk(os.path.join( + self.upload_path, job_directory + )): + for filename in filenames: + artifacts[job_directory].append(os.path.join( + dirpath, filename + )) + + for job_name in jobs: + report = getUtility(IRevisionStatusReportSet).getByCIBuildAndTitle( + build, job_name) + if not report: + # the report should normally exist, since the build request + # logic will eventually create report rows for the jobs it + # expects to run, but for robustness it's a good idea to + # ensure its existence here + report = getUtility(IRevisionStatusReportSet).new( + creator=build.git_repository.owner, + title=job_name, + git_repository=build.git_repository, + commit_sha1=build.commit_sha1, + ci_build=build, + ) + + # attach log file + log_file = os.path.join(self.upload_path, job_name + ".log") + with open(log_file, mode="rb") as f: + report.setLog(f.read()) + + # attach artifacts + for file_path in artifacts[job_name]: + with open(file_path, mode="rb") as f: + report.attach( + name=os.path.basename(file_path), data=f.read() + ) + + # set status + report.update( + result=getattr(RevisionStatusResult, jobs[job_name]["result"]) + ) + + self.logger.debug("Updating %s" % build.title) + build.updateStatus(BuildStatus.FULLYBUILT) + + self.logger.debug("Finished upload.") diff --git a/lib/lp/archiveuploader/tests/test_ciupload.py b/lib/lp/archiveuploader/tests/test_ciupload.py new file mode 100644 index 0000000..d0f0a11 --- /dev/null +++ b/lib/lp/archiveuploader/tests/test_ciupload.py @@ -0,0 +1,135 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Test uploads of CIBuilds.""" + +import json +import os + +from storm.store import Store +from zope.component import getUtility + +from lp.archiveuploader.tests.test_uploadprocessor import ( + TestUploadProcessorBase, + ) +from lp.archiveuploader.uploadprocessor import ( + UploadHandler, + UploadStatusEnum, + ) +from lp.buildmaster.enums import BuildStatus +from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet +from lp.services.osutils import write_file + + +class TestCIUBuildUploads(TestUploadProcessorBase): + """End-to-end tests of CIBuild uploads.""" + + def setUp(self): + super().setUp() + self.switchToAdmin() + self.build = self.factory.makeCIBuild() + self.build.updateStatus(BuildStatus.UPLOADING) + Store.of(self.build).flush() + self.switchToUploader() + self.uploadprocessor = self.getUploadProcessor( + self.layer.txn, builds=True + ) + + def test_requires_CI_job(self): + """If no jobs run, no `jobs.json` will be created. + + This results in an `UploadError` / rejected upload.""" + handler = UploadHandler.forProcessor( + self.uploadprocessor, + self.incoming_folder, + "test", + self.build, + ) + + result = handler.processCIResult(self.log) + + self.assertEqual( + UploadStatusEnum.REJECTED, result + ) + + def test_triggers_store_upload_for_ci(self): + # create "jobs.json" + path = os.path.join(self.incoming_folder, "test", "jobs.json") + content = { + 'build:0': + { + 'log': 'test_file_hash', + 'result': 'SUCCEEDED', + } + } + write_file(path, json.dumps(content).encode("utf-8")) + + # create log file + path = os.path.join(self.incoming_folder, "test", "build:0.log") + content = "some log content" + write_file(path, content.encode("utf-8")) + + # create artifact + path = os.path.join( + self.incoming_folder, "test", "build:0", "ci.whl") + content = b"abc" + write_file(path, content) + + revision_status_report = self.factory.makeRevisionStatusReport( + title="build:0", + ci_build=self.build, + ) + Store.of(revision_status_report).flush() + + handler = UploadHandler.forProcessor( + self.uploadprocessor, + self.incoming_folder, + "test", + self.build, + ) + + result = handler.processCIResult(self.log) + + self.assertEqual(UploadStatusEnum.ACCEPTED, result) + self.assertEqual(BuildStatus.FULLYBUILT, self.build.status) + + def test_creates_revision_status_report_if_not_present(self): + # create "jobs.json" + path = os.path.join(self.incoming_folder, "test", "jobs.json") + content = { + 'build:0': + { + 'log': 'test_file_hash', + 'result': 'SUCCEEDED', + } + } + write_file(path, json.dumps(content).encode("utf-8")) + + # create log file + path = os.path.join(self.incoming_folder, "test", "build:0.log") + content = "some log content" + write_file(path, content.encode("utf-8")) + + # create artifact + path = os.path.join( + self.incoming_folder, "test", "build:0", "ci.whl") + content = b"abc" + write_file(path, content) + + handler = UploadHandler.forProcessor( + self.uploadprocessor, + self.incoming_folder, + "test", + self.build, + ) + + result = handler.processCIResult(self.log) + + self.assertEqual( + self.build, + getUtility( + IRevisionStatusReportSet + ).getByCIBuildAndTitle(self.build, "build:0").ci_build + ) + self.assertEqual(UploadStatusEnum.ACCEPTED, result) + self.assertEqual(BuildStatus.FULLYBUILT, self.build.status) diff --git a/lib/lp/archiveuploader/uploadprocessor.py b/lib/lp/archiveuploader/uploadprocessor.py index b71d0aa..3b6820d 100644 --- a/lib/lp/archiveuploader/uploadprocessor.py +++ b/lib/lp/archiveuploader/uploadprocessor.py @@ -53,6 +53,7 @@ from zope.component import getUtility from lp.app.errors import NotFoundError from lp.archiveuploader.charmrecipeupload import CharmRecipeUpload +from lp.archiveuploader.ciupload import CIUpload from lp.archiveuploader.livefsupload import LiveFSUpload from lp.archiveuploader.nascentupload import ( EarlyReturnUploadError, @@ -68,6 +69,7 @@ from lp.archiveuploader.utils import UploadError from lp.buildmaster.enums import BuildStatus from lp.buildmaster.interfaces.buildfarmjob import ISpecificBuildFarmJobSource from lp.charms.interfaces.charmrecipebuild import ICharmRecipeBuild +from lp.code.interfaces.cibuild import ICIBuild from lp.code.interfaces.sourcepackagerecipebuild import ( ISourcePackageRecipeBuild, ) @@ -584,6 +586,31 @@ class BuildUploadHandler(UploadHandler): "Unable to find %s with id %d. Skipping." % (job_type, job_id)) + def processCIResult(self, logger=None): + """Process a CI result upload.""" + assert ICIBuild.providedBy(self.build) + if logger is None: + logger = self.processor.log + try: + logger.info("Processing CI result upload %s" % self.upload_path) + CIUpload(self.upload_path, logger).process(self.build) + + if self.processor.dry_run: + logger.info("Dry run, aborting transaction.") + self.processor.ztm.abort() + else: + logger.info( + "Committing the transaction and any mails associated " + "with this upload.") + self.processor.ztm.commit() + return UploadStatusEnum.ACCEPTED + except UploadError as e: + logger.error(str(e)) + return UploadStatusEnum.REJECTED + except BaseException: + self.processor.ztm.abort() + raise + def processLiveFS(self, logger=None): """Process a live filesystem upload.""" assert ILiveFSBuild.providedBy(self.build) @@ -727,6 +754,8 @@ class BuildUploadHandler(UploadHandler): result = self.processOCIRecipe(logger) elif ICharmRecipeBuild.providedBy(self.build): result = self.processCharmRecipe(logger) + elif ICIBuild.providedBy(self.build): + result = self.processCIResult(logger) else: self.processor.log.debug("Build %s found" % self.build.id) [changes_file] = self.locateChangesFiles() diff --git a/lib/lp/code/interfaces/revisionstatus.py b/lib/lp/code/interfaces/revisionstatus.py index 7db816b..90d4e02 100644 --- a/lib/lp/code/interfaces/revisionstatus.py +++ b/lib/lp/code/interfaces/revisionstatus.py @@ -231,6 +231,9 @@ class IRevisionStatusReportSet(Interface): def findByCommit(repository, commit_sha1): """Returns all `RevisionStatusReport` for a repository and commit.""" + def getByCIBuildAndTitle(ci_build, title): + """Return the `RevisionStatusReport` for a given CI build and title.""" + def deleteForRepository(repository): """Delete all `RevisionStatusReport` for a repository.""" diff --git a/lib/lp/code/model/revisionstatus.py b/lib/lp/code/model/revisionstatus.py index 624fa77..8ef5cba 100644 --- a/lib/lp/code/model/revisionstatus.py +++ b/lib/lp/code/model/revisionstatus.py @@ -170,6 +170,11 @@ class RevisionStatusReportSet: RevisionStatusReport.date_created, RevisionStatusReport.id) + def getByCIBuildAndTitle(self, ci_build, title): + """See `IRevisionStatusReportSet`.""" + return IStore(RevisionStatusReport).find( + RevisionStatusReport, ci_build=ci_build, title=title).one() + def deleteForRepository(self, repository): clauses = [ RevisionStatusArtifact.report == RevisionStatusReport.id,
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp