dexter has uploaded this change for review. ( https://gerrit.osmocom.org/c/pysim/+/41874?usp=email )
Change subject: esim/http_json_api: extend JSON API with server functionality ...................................................................... esim/http_json_api: extend JSON API with server functionality At the moment http_json_api only supports the client role. Let's also add support for the server role. This patch refactors the existing client code. This in particular means that the following preperations have to be made: - To use the existing JsonHttpApiFunction definitions in the client and server the scheme has to be symetric. It already is for the most part, but it treads the header field differently. So let's just tread the header field like any other mandatory field and add it input_params. (this does not affect the es9p.py code since in ES9+ the requests have no header messages, see also SGP.22, section 6.5.1.1) - The JsonHttpApiFunction class currently also has the code to perform the client requests. Let's seperate that code in a JsonHttpApiClient class to which we pass an JsonHttpApiFunction object. - The code that does the encoding and decoding in the client role has lots of conditions the tread the header differently. Let's do the decisions about the header in the JsonHttpApiClient. The encoder and decoder function should do the generic encoding and decoding only. (however, some generic header specific conditions will remain). The code for the server role logically mirrors the code for the client role. We add a JsonHttpApiServer class that can be used to create API endpoints. The API user has to pass in a call_handler through which the application logic is defined. Above that we add an Es2pApiServer class in es2p. In this class we implement the logic that runs the HTTP server and receives the requests. The Es2pApiServer supports all ES2+ functions defined by GSMA SGP.22. The user may use the provided Es2pApiServerHandler base class to define the application logic for each ES2+ function. Related: SYS#7825 Change-Id: I277aa90fddb5171c4bf6c3436259aa371d30d092 --- M pySim/esim/es2p.py M pySim/esim/http_json_api.py 2 files changed, 293 insertions(+), 44 deletions(-) git pull ssh://gerrit.osmocom.org:29418/pysim refs/changes/74/41874/1 diff --git a/pySim/esim/es2p.py b/pySim/esim/es2p.py index b917b47..9aa1764 100644 --- a/pySim/esim/es2p.py +++ b/pySim/esim/es2p.py @@ -16,6 +16,12 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import requests +from klein import Klein +from twisted.internet import defer, protocol, ssl, task, endpoints, reactor +from twisted.internet.posixbase import PosixReactorBase +from pathlib import Path +from twisted.web.server import Site, Request + import logging from datetime import datetime import time @@ -123,10 +129,12 @@ class DownloadOrder(Es2PlusApiFunction): path = '/gsma/rsp2/es2plus/downloadOrder' input_params = { + 'header': JsonRequestHeader, 'eid': param.Eid, 'iccid': param.Iccid, 'profileType': param.ProfileType } + input_mandatory = ['header'] output_params = { 'header': JsonResponseHeader, 'iccid': param.Iccid, @@ -137,6 +145,7 @@ class ConfirmOrder(Es2PlusApiFunction): path = '/gsma/rsp2/es2plus/confirmOrder' input_params = { + 'header': JsonRequestHeader, 'iccid': param.Iccid, 'eid': param.Eid, 'matchingId': param.MatchingId, @@ -144,7 +153,7 @@ 'smdsAddress': param.SmdsAddress, 'releaseFlag': param.ReleaseFlag, } - input_mandatory = ['iccid', 'releaseFlag'] + input_mandatory = ['header', 'iccid', 'releaseFlag'] output_params = { 'header': JsonResponseHeader, 'eid': param.Eid, @@ -157,12 +166,13 @@ class CancelOrder(Es2PlusApiFunction): path = '/gsma/rsp2/es2plus/cancelOrder' input_params = { + 'header': JsonRequestHeader, 'iccid': param.Iccid, 'eid': param.Eid, 'matchingId': param.MatchingId, 'finalProfileStatusIndicator': param.FinalProfileStatusIndicator, } - input_mandatory = ['finalProfileStatusIndicator', 'iccid'] + input_mandatory = ['header', 'finalProfileStatusIndicator', 'iccid'] output_params = { 'header': JsonResponseHeader, } @@ -172,9 +182,10 @@ class ReleaseProfile(Es2PlusApiFunction): path = '/gsma/rsp2/es2plus/releaseProfile' input_params = { + 'header': JsonRequestHeader, 'iccid': param.Iccid, } - input_mandatory = ['iccid'] + input_mandatory = ['header', 'iccid'] output_params = { 'header': JsonResponseHeader, } @@ -184,6 +195,7 @@ class HandleDownloadProgressInfo(Es2PlusApiFunction): path = '/gsma/rsp2/es2plus/handleDownloadProgressInfo' input_params = { + 'header': JsonRequestHeader, 'eid': param.Eid, 'iccid': param.Iccid, 'profileType': param.ProfileType, @@ -192,10 +204,9 @@ 'notificationPointStatus': param.NotificationPointStatus, 'resultData': param.ResultData, } - input_mandatory = ['iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus'] + input_mandatory = ['header', 'iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus'] expected_http_status = 204 - class Es2pApiClient: """Main class representing a full ES2+ API client. Has one method for each API function.""" def __init__(self, url_prefix:str, func_req_id:str, server_cert_verify: str = None, client_cert: str = None): @@ -206,18 +217,17 @@ if client_cert: self.session.cert = client_cert - self.downloadOrder = DownloadOrder(url_prefix, func_req_id, self.session) - self.confirmOrder = ConfirmOrder(url_prefix, func_req_id, self.session) - self.cancelOrder = CancelOrder(url_prefix, func_req_id, self.session) - self.releaseProfile = ReleaseProfile(url_prefix, func_req_id, self.session) - self.handleDownloadProgressInfo = HandleDownloadProgressInfo(url_prefix, func_req_id, self.session) + self.downloadOrder = JsonHttpApiClient(DownloadOrder(), url_prefix, func_req_id, self.session) + self.confirmOrder = JsonHttpApiClient(ConfirmOrder(), url_prefix, func_req_id, self.session) + self.cancelOrder = JsonHttpApiClient(CancelOrder(), url_prefix, func_req_id, self.session) + self.releaseProfile = JsonHttpApiClient(ReleaseProfile(), url_prefix, func_req_id, self.session) + self.handleDownloadProgressInfo = JsonHttpApiClient(HandleDownloadProgressInfo(), url_prefix, func_req_id, self.session) def _gen_func_id(self) -> str: """Generate the next function call id.""" self.func_id += 1 return 'FCI-%u-%u' % (time.time(), self.func_id) - def call_downloadOrder(self, data: dict) -> dict: """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1).""" return self.downloadOrder.call(data, self._gen_func_id()) @@ -237,3 +247,93 @@ def call_handleDownloadProgressInfo(self, data: dict) -> dict: """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5).""" return self.handleDownloadProgressInfo.call(data, self._gen_func_id()) + +class Es2pApiServerHandler(): + """ES2+ API Server handler class. The API user is expected to override the contained methods as needed.""" + + def call_downloadOrder(self, data: dict) -> (dict, str): + """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1).""" + return {}, 'Failed' + + def call_confirmOrder(self, data: dict) -> (dict, str): + """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2).""" + return {}, 'Failed' + + def call_cancelOrder(self, data: dict) -> (dict, str): + """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3).""" + return {}, 'Failed' + + def call_releaseProfile(self, data: dict) -> (dict, str): + """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4).""" + return {}, 'Failed' + + def call_handleDownloadProgressInfo(self, data: dict) -> (dict, str): + """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5).""" + return {}, 'Failed' + +class Es2pApiServer: + """Main class representing a full ES2+ API server. Has one method for each API function.""" + app = Klein() + + def __init__(self, port: int, interface: str, handler: Es2pApiServerHandler, + server_cert: str = None, client_cert_verify: str = None): + logger.debug("HTTP SRV: starting ES2+ API server on %s:%s" % (interface, port)) + + self.port = port + self.interface = interface + self.handler = handler + if server_cert: + self.server_cert = ssl.PrivateCertificate.loadPEM(Path(server_cert).read_text()) + else: + self.server_cert = None + if client_cert_verify: + self.client_cert_verify = ssl.Certificate.loadPEM(Path(client_cert_verify).read_text()) + else: + self.client_cert_verify = None + + self.downloadOrder = JsonHttpApiServer(DownloadOrder(), handler.call_downloadOrder) + self.confirmOrder = JsonHttpApiServer(ConfirmOrder(), handler.call_confirmOrder) + self.cancelOrder = JsonHttpApiServer(CancelOrder(), handler.call_cancelOrder) + self.releaseProfile = JsonHttpApiServer(ReleaseProfile(), handler.call_releaseProfile) + self.handleDownloadProgressInfo = JsonHttpApiServer(HandleDownloadProgressInfo(), + handler.call_handleDownloadProgressInfo) + + task.react(self.reactor) + + def reactor(self, reactor: PosixReactorBase): + logger.debug("HTTP SRV: listen on %s:%s" % (self.interface, self.port)) + if self.server_cert: + if self.client_cert_verify: + reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(self.client_cert_verify), + interface=self.interface) + else: + reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(), + interface=self.interface) + else: + reactor.listenTCP(self.port, Site(self.app.resource()), interface=self.interface) + return defer.Deferred() + + @app.route(DownloadOrder.path) + def call_downloadOrder(self, request: Request) -> dict: + """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1).""" + return self.downloadOrder.call(request) + + @app.route(ConfirmOrder.path) + def call_confirmOrder(self, request: Request) -> dict: + """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2).""" + return self.confirmOrder.call(request) + + @app.route(CancelOrder.path) + def call_cancelOrder(self, request: Request) -> dict: + """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3).""" + return self.cancelOrder.call(request) + + @app.route(ReleaseProfile.path) + def call_releaseProfile(self, request: Request) -> dict: + """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4).""" + return self.releaseProfile.call(request) + + @app.route(HandleDownloadProgressInfo.path) + def call_handleDownloadProgressInfo(self, request: Request) -> dict: + """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5).""" + return self.handleDownloadProgressInfo.call(request) diff --git a/pySim/esim/http_json_api.py b/pySim/esim/http_json_api.py index bb41df3..1081f91 100644 --- a/pySim/esim/http_json_api.py +++ b/pySim/esim/http_json_api.py @@ -21,6 +21,8 @@ import json from typing import Optional import base64 +from twisted.web.server import Request + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -131,6 +133,16 @@ if status not in ['Executed-Success', 'Executed-WithWarning', 'Failed', 'Expired']: raise ValueError('Unknown/unspecified status "%s"' % status) +class JsonRequestHeader(ApiParam): + """SGP.22 section 6.5.1.3.""" + @classmethod + def verify_decoded(cls, data): + func_req_id = data.get('functionRequesterIdentifier') + if not func_req_id: + raise ValueError('Missing mandatory functionRequesterIdentifier in header') + func_call_id = data.get('functionCallIdentifier') + if not func_call_id: + raise ValueError('Missing mandatory functionCallIdentifier in header') class HttpStatusError(Exception): pass @@ -161,36 +173,46 @@ class JsonHttpApiFunction(abc.ABC): """Base class for representing an HTTP[s] API Function.""" - # the below class variables are expected to be overridden in derived classes + # The below class variables are used to describe the properties of the API function. Derived classes are expected + # to orverride those class properties with useful values. The prefixes "input_" and "output_" refer to the API + # function from an abstract point of view. Seen from the client perspective, "input_" will refer to parameters the + # client sends to a HTTP server. Seen from the server perspective, "input_" will refer to parameters the server + # receives from the a requesting client. The same applies vice versa to class variables that have an "output_" + # prefix. + # path of the API function (e.g. '/gsma/rsp2/es2plus/confirmOrder') path = None + # dictionary of input parameters. key is parameter name, value is ApiParam class input_params = {} + # list of mandatory input parameters input_mandatory = [] + # dictionary of output parameters. key is parameter name, value is ApiParam class output_params = {} + # list of mandatory output parameters (for successful response) output_mandatory = [] + + # list of mandatory output parameters (for failed response) + output_mandatory_failed = [] + # expected HTTP status code of the response expected_http_status = 200 + # the HTTP method used (GET, OPTIONS, HEAD, POST, PUT, PATCH or DELETE) http_method = 'POST' + + # additional custom HTTP headers (client requests) extra_http_req_headers = {} - def __init__(self, url_prefix: str, func_req_id: Optional[str], session: requests.Session): - self.url_prefix = url_prefix - self.func_req_id = func_req_id - self.session = session + # additional custom HTTP headers (server responses) + extra_http_res_headers = {} - def encode(self, data: dict, func_call_id: Optional[str] = None) -> dict: + def encode_client(self, data: dict) -> dict: """Validate an encode input dict into JSON-serializable dict for request body.""" output = {} - if func_call_id: - output['header'] = { - 'functionRequesterIdentifier': self.func_req_id, - 'functionCallIdentifier': func_call_id - } for p in self.input_mandatory: if not p in data: @@ -204,22 +226,19 @@ output[p] = p_class.encode(v) return output - def decode(self, data: dict) -> dict: + def decode_client(self, data: dict) -> dict: """[further] Decode and validate the JSON-Dict of the response body.""" output = {} - if 'header' in self.output_params: - # let's first do the header, it's special - if not 'header' in data: - raise ValueError('Mandatory output parameter "header" missing') - hdr_class = self.output_params.get('header') - output['header'] = hdr_class.decode(data['header']) + output_mandatory = self.output_mandatory - if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']: - raise ApiError(output['header']['functionExecutionStatus']) - # we can only expect mandatory parameters to be present in case of successful execution - for p in self.output_mandatory: - if p == 'header': - continue + # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a + # different set of mandatory parameters applies. + header = data.get('header') + if header: + if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']: + output_mandatory = self.output_mandatory_failed + + for p in output_mandatory: if not p in data: raise ValueError('Mandatory output parameter "%s" missing' % p) for p, v in data.items(): @@ -231,30 +250,160 @@ output[p] = p_class.decode(v) return output + def encode_server(self, data: dict) -> dict: + """Validate an encode input dict into JSON-serializable dict for response body.""" + output = {} + output_mandatory = self.output_mandatory + + # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a + # different set of mandatory parameters applies. + header = data.get('header') + if header: + if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']: + output_mandatory = self.output_mandatory_failed + + for p in output_mandatory: + if not p in data: + raise ValueError('Mandatory output parameter %s missing' % p) + for p, v in data.items(): + p_class = self.output_params.get(p) + if not p_class: + logger.warning('Unexpected/unsupported output parameter %s=%s', p, v) + output[p] = v + else: + output[p] = p_class.encode(v) + return output + + def decode_server(self, data: dict) -> dict: + """[further] Decode and validate the JSON-Dict of the request body.""" + output = {} + + for p in self.input_mandatory: + if not p in data: + raise ValueError('Mandatory input parameter "%s" missing' % p) + for p, v in data.items(): + p_class = self.input_params.get(p) + if not p_class: + logger.warning('Unexpected/unsupported input parameter "%s"="%s"', p, v) + output[p] = v + else: + output[p] = p_class.decode(v) + return output + +class JsonHttpApiClient(): + def __init__(self, api_func: JsonHttpApiFunction, url_prefix: str, func_req_id: Optional[str], + session: requests.Session): + self.api_func = api_func + self.url_prefix = url_prefix + self.func_req_id = func_req_id + self.session = session + def call(self, data: dict, func_call_id: Optional[str] = None, timeout=10) -> Optional[dict]: - """Make an API call to the HTTP API endpoint represented by this object. - Input data is passed in `data` as json-serializable dict. Output data - is returned as json-deserialized dict.""" - url = self.url_prefix + self.path - encoded = json.dumps(self.encode(data, func_call_id)) + """Make an API call to the HTTP API endpoint represented by this object. Input data is passed in `data` as + json-serializable dict. Output data is returned as json-deserialized dict.""" + + # In case a function caller ID is supplied, use it together with the stored function requestor ID to generate + # and prepend the header field according to SGP.22, section 6.5.1.1 and 6.5.1.3. (the presence of the header + # field is checked by the encode_client method) + if func_call_id: + data = {'header' : {'functionRequesterIdentifier': self.func_req_id, + 'functionCallIdentifier': func_call_id}} | data + + # Encode the message (the presence of mandatory fields is checked during encoding) + encoded = json.dumps(self.api_func.encode_client(data)) + + # Apply HTTP request headers according to SGP.22, section 6.5.1 req_headers = { 'Content-Type': 'application/json', 'X-Admin-Protocol': 'gsma/rsp/v2.5.0', } - req_headers.update(self.extra_http_req_headers) + req_headers.update(self.api_func.extra_http_req_headers) + # Perform HTTP request + url = self.url_prefix + self.api_func.path logger.debug("HTTP REQ %s - hdr: %s '%s'" % (url, req_headers, encoded)) - response = self.session.request(self.http_method, url, data=encoded, headers=req_headers, timeout=timeout) + response = self.session.request(self.api_func.http_method, url, data=encoded, headers=req_headers, timeout=timeout) logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers)) logger.debug("HTTP RSP: %s" % (response.content)) - if response.status_code != self.expected_http_status: + # Check HTTP response status code and make sure that the returned HTTP headers look plausible (according to + # SGP.22, section 6.5.1) + if response.status_code != self.api_func.expected_http_status: raise HttpStatusError(response) if not response.headers.get('Content-Type').startswith(req_headers['Content-Type']): raise HttpHeaderError(response) if not response.headers.get('X-Admin-Protocol', 'gsma/rsp/v2.unknown').startswith('gsma/rsp/v2.'): raise HttpHeaderError(response) + # Decode response and return the result back to the caller if response.content: - return self.decode(response.json()) + output = self.api_func.decode_client(response.json()) + # In case the response contains a header, check it to make sure that the API call was executed successfully + # (the presence of the header field is checked by the decode_client method) + if 'header' in output: + if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']: + raise ApiError(output['header']['functionExecutionStatus']) + return output return None + +class JsonHttpApiServer(): + def __init__(self, api_func: JsonHttpApiFunction, call_handler = None): + """ + Args: + api_func : API function definition (JsonHttpApiFunction) + call_handler : handler function to process the request. This function must accept the + decoded request as a dictionary. The handler function must return a tuple consisting + of the response in the form of a dictionary (may be empty), and a function execution + status string ('Executed-Success', 'Executed-WithWarning', 'Failed' or 'Expired') + """ + self.api_func = api_func + if call_handler: + self.call_handler = call_handler + else: + self.call_handler = self.default_handler + + def default_handler(self, data: dict) -> (dict, str): + """default handler, used in case no call handler is provided.""" + logger.error("no handler function for request: %s" % str(data)) + return {}, 'Failed' + + def call(self, request: Request) -> str: + """ Process an incoming request. + Args: + request : request object as received using twisted.web.server + Returns: + encoded JSON string (HTTP response code and headers are set by calling the apropriate methods on the + provided the request object) + """ + + # Make sure the request is done with the correct HTTP method + if (request.method.decode() != self.api_func.http_method): + raise ValueError('Wrong HTTP method %s!=%s' % (request.method.decode(), self.api_func.http_method)) + + # Decode the request + decoded_request = self.api_func.decode_server(json.loads(request.content.read())) + + # Run call handler (see above) + data, fe_status = self.call_handler(decoded_request) + + # In case a function execution status is returned, use it to generate and prepend the header field according to + # SGP.22, section 6.5.1.2 and 6.5.1.4 (the presence of the header filed is checked by the encode_server method) + if fe_status: + data = {'header' : {'functionExecutionStatus': {'status' : fe_status}}} | data + + # Encode the message (the presence of mandatory fields is checked during encoding) + encoded = json.dumps(self.api_func.encode_server(data)) + + # Apply HTTP request headers according to SGP.22, section 6.5.1 + res_headers = { + 'Content-Type': 'application/json', + 'X-Admin-Protocol': 'gsma/rsp/v2.5.0', + } + res_headers.update(self.api_func.extra_http_res_headers) + for header, value in res_headers.items(): + request.setHeader(header, value) + request.setResponseCode(self.api_func.expected_http_status) + + # Return the encoded result back to the caller for sending (using twisted/klein) + return encoded + -- To view, visit https://gerrit.osmocom.org/c/pysim/+/41874?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: pysim Gerrit-Branch: master Gerrit-Change-Id: I277aa90fddb5171c4bf6c3436259aa371d30d092 Gerrit-Change-Number: 41874 Gerrit-PatchSet: 1 Gerrit-Owner: dexter <[email protected]>
