Simone Pelosi has proposed merging ~pelpsi/launchpad:request-token-and-session-from-fetch-service into launchpad:master with ~ines-almeida/launchpad:fetch-service-option-model-update as a prerequisite.
Commit message: Request fetch service session NOTE: THIS IS A WIP! This message will be updated! Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~pelpsi/launchpad/+git/launchpad/+merge/461586 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~pelpsi/launchpad:request-token-and-session-from-fetch-service into launchpad:master.
diff --git a/lib/lp/buildmaster/builderproxy.py b/lib/lp/buildmaster/builderproxy.py index be65e29..6857305 100644 --- a/lib/lp/buildmaster/builderproxy.py +++ b/lib/lp/buildmaster/builderproxy.py @@ -20,7 +20,10 @@ from typing import Dict, Generator from twisted.internet import defer -from lp.buildmaster.downloader import RequestProxyTokenCommand +from lp.buildmaster.downloader import ( + RequestFetchServiceSessionCommand, + RequestProxyTokenCommand, +) from lp.buildmaster.interfaces.builder import CannotBuild from lp.buildmaster.interfaces.buildfarmjobbehaviour import BuildArgs from lp.services.config import config @@ -36,9 +39,16 @@ class BuilderProxyMixin: @defer.inlineCallbacks def addProxyArgs( - self, args: BuildArgs, allow_internet: bool = True + self, + args: BuildArgs, + allow_internet: bool = True, + fetch_service: bool = False, ) -> Generator[None, Dict[str, str], None]: - if _get_proxy_config("builder_proxy_host") and allow_internet: + if ( + not fetch_service + and _get_proxy_config("builder_proxy_host") + and allow_internet + ): token = yield self._requestProxyToken() args["proxy_url"] = ( "http://{username}:{password}@{host}:{port}".format( @@ -52,6 +62,25 @@ class BuilderProxyMixin: endpoint=_get_proxy_config("builder_proxy_auth_api_endpoint"), token=token["username"], ) + if ( + fetch_service + and _get_proxy_config("fetch_service_host") + and allow_internet + ): + # http://<session-id>:<secret>@<addr>:9988/ + token = yield self._requestFetchServiceSession() + args["proxy_url"] = ( + "http://{session_id}:{token}@{host}:{port}".format( + session_id=token["id"], + token=token["token"], + host=_get_proxy_config("fetch_service_host"), + port=_get_proxy_config("fetch_service_port"), + ) + ) + args["revocation_endpoint"] = "{endpoint}/{token}".format( + endpoint=_get_proxy_config("fetch_service_auth_api_endpoint"), + token=token["username"], + ) @defer.inlineCallbacks def _requestProxyToken(self): @@ -86,3 +115,39 @@ class BuilderProxyMixin: proxy_username=proxy_username, ) return token + + @defer.inlineCallbacks + def _requestFetchServiceSession(self): + # This is a different function if we have the needs to + # differentiate more the behavior. + admin_username = _get_proxy_config( + "fetch_service_auth_api_admin_username" + ) + if not admin_username: + raise CannotBuild( + "fetch_service_auth_api_admin_username is not configured." + ) + secret = _get_proxy_config("fetch_service_auth_api_admin_secret") + if not secret: + raise CannotBuild( + "fetch_service_auth_api_admin_secret is not configured." + ) + url = _get_proxy_config("fetch_service_auth_api_endpoint") + if not secret: + raise CannotBuild( + "fetch_service_auth_api_endpoint is not configured." + ) + timestamp = int(time.time()) + proxy_username = "{build_id}-{timestamp}".format( + build_id=self.build.build_cookie, timestamp=timestamp + ) + auth_string = f"{admin_username}:{secret}".strip() + auth_header = b"Basic " + base64.b64encode(auth_string.encode("ASCII")) + + token = yield self._worker.process_pool.doWork( + RequestFetchServiceSessionCommand, + url=url, + auth_header=auth_header, + proxy_username=proxy_username, + ) + return token diff --git a/lib/lp/buildmaster/downloader.py b/lib/lp/buildmaster/downloader.py index fc16852..ecc10f9 100644 --- a/lib/lp/buildmaster/downloader.py +++ b/lib/lp/buildmaster/downloader.py @@ -10,6 +10,7 @@ anything from the rest of Launchpad. __all__ = [ "DownloadCommand", "RequestProcess", + "RequestFetchServiceSessionCommand", "RequestProxyTokenCommand", ] @@ -37,6 +38,21 @@ class DownloadCommand(amp.Command): } +class RequestFetchServiceSessionCommand(amp.Command): + arguments = [ + (b"url", amp.Unicode()), + (b"auth_header", amp.String()), + (b"proxy_username", amp.Unicode()), + ] + response = [ + (b"id", amp.Unicode()), + (b"token", amp.Unicode()), + ] + errors = { + RequestException: b"REQUEST_ERROR", + } + + class RequestProxyTokenCommand(amp.Command): arguments = [ (b"url", amp.Unicode()), @@ -94,3 +110,23 @@ class RequestProcess(AMPChild): ) response.raise_for_status() return response.json() + + @RequestFetchServiceSessionCommand.responder + def requestFetchServiceSessionCommand( + self, url, auth_header, proxy_username + ): + with Session() as session: + session.trust_env = False + # XXX pelpsi 2024-02-28: should take the same + # json body? From ST108 we should provide + # { + # "timeout": <int>, // session timeout in seconds + # "policy": <string> // "strict" or "permissive" + # } + response = session.post( + url, + headers={"Authorization": auth_header}, + json={"username": proxy_username}, + ) + response.raise_for_status() + return response.json() diff --git a/lib/lp/snappy/model/snapbuildbehaviour.py b/lib/lp/snappy/model/snapbuildbehaviour.py index 66ae7cc..58fdd5f 100644 --- a/lib/lp/snappy/model/snapbuildbehaviour.py +++ b/lib/lp/snappy/model/snapbuildbehaviour.py @@ -116,7 +116,9 @@ class SnapBuildBehaviour(BuilderProxyMixin, BuildFarmJobBehaviourBase): """ build: ISnapBuild = self.build args: BuildArgs = yield super().extraBuildArgs(logger=logger) - yield self.addProxyArgs(args, build.snap.allow_internet) + yield self.addProxyArgs( + args, build.snap.allow_internet, build.snap.use_fetch_service + ) args["name"] = build.snap.store_name or build.snap.name channels = build.channels or {} if "snapcraft" not in channels:
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp