Vinzenz Feenstra has uploaded a new change for review. Change subject: Allow GZIP transport on XMLRPC ......................................................................
Allow GZIP transport on XMLRPC Change-Id: Iebdaa04b17e2c1df1c1852ed536c5d6d8ec8d88b Signed-off-by: Vinzenz Feenstra <[email protected]> --- M lib/vdsm/utils.py M lib/vdsm/vdscli.py.in 2 files changed, 113 insertions(+), 2 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/01/25201/1 diff --git a/lib/vdsm/utils.py b/lib/vdsm/utils.py index b682dec..81397ef 100644 --- a/lib/vdsm/utils.py +++ b/lib/vdsm/utils.py @@ -37,6 +37,7 @@ import fcntl import functools import glob +import gzip import io import itertools import logging @@ -44,6 +45,7 @@ import os import platform import pwd +import re import select import shutil import signal @@ -148,6 +150,28 @@ raise +## +# Encode a string using the gzip content encoding such as specified by the +# Content-Encoding: gzip +# in the HTTP header, as described in RFC 1952 +# +# @param data the unencoded data +# @return the encoded data + +def gzip_encode(data): + """data -> gzip encoded data + + Encode data using the gzip content encoding as described in RFC 1952 + """ + f = StringIO() + gzf = gzip.GzipFile(mode="wb", fileobj=f, compresslevel=1) + gzf.write(data) + gzf.close() + encoded = f.getvalue() + f.close() + return encoded + + class IPXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): if config.getboolean('vars', 'xmlrpc_http11'): @@ -171,6 +195,26 @@ # the methods only on Python 2.6. if sys.version_info[:2] == (2, 6): + def __init__(self, *args, **kwargs): + self.encode_threshold = 1400 + SimpleXMLRPCRequestHandler.__init__(self, *args, **kwargs) + + # a re to match a gzip Accept-Encoding + aepattern = re.compile(r""" + \s* ([^\s;]+) \s* #content-coding + (;\s* q \s*=\s* ([0-9\.]+))? #q + """, re.VERBOSE | re.IGNORECASE) + + def accept_encodings(self): + r = {} + ae = self.headers.get("Accept-Encoding", "") + for e in ae.split(","): + match = self.aepattern.match(e) + if match: + v = match.group(3) + v = float(v) if v else 1.0 + r[match.group(1)] = v + return r def do_POST(self): # Check that the path is legal @@ -183,7 +227,7 @@ # We read this in chunks to avoid straining # socket.read(); around the 10 or 15Mb mark, some platforms # begin to have problems (bug #792570). - max_chunk_size = 10*1024*1024 + max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: @@ -218,9 +262,20 @@ # got a valid XML RPC response self.send_response(200) self.send_header("Content-type", "text/xml") + pureLen = len(response) + if self.encode_threshold is not None: + if pureLen > self.encode_threshold: + q = self.accept_encodings().get("gzip", 0) + if q: + response = gzip_encode(response) + self.send_header("Content-encoding", "gzip") self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) + import datetime + import re + with open('/tmp/rpc-stats.log', 'a') as statslog: + statslog.write("%s - %s - %d - %d\n" % (datetime.datetime.utcnow().isoformat(), re.search(r'(?<=<methodName>)(.+)(?=<\/methodName)', data).group(0), len(response), pureLen)) def report_404(self): self.send_response(404) diff --git a/lib/vdsm/vdscli.py.in b/lib/vdsm/vdscli.py.in index 5fa7528..e9e5c26 100644 --- a/lib/vdsm/vdscli.py.in +++ b/lib/vdsm/vdscli.py.in @@ -19,10 +19,12 @@ # Refer to the README and COPYING files for full details of the license # +import gzip import xmlrpclib import subprocess import os import re +import StringIO import sys from xml.parsers.expat import ExpatError from . import SecureXMLRPCServer @@ -34,7 +36,61 @@ d_port = '54321' +if sys.version_info[:2] == (2, 6): + class GzipDecodedResponse(gzip.GzipFile if gzip else object): + """a file-like object to decode a response encoded with the gzip + method, as described in RFC 1952. + """ + def __init__(self, response): + data = response.read() + with open("/tmp/client.dump", "wb") as f: + f.write(data) + self.stringio = StringIO.StringIO(data) + gzip.GzipFile.__init__(self, mode="rb", fileobj=self.stringio) + + def close(self): + gzip.GzipFile.close(self) + self.stringio.close() + + def wrap_request(transport): + self = transport + + def wrapped_request(host, handler, request_body, verbose=0): + # issue XML-RPC request + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + h.putheader("Accept-Encoding", "gzip") + self.send_content(h, request_body) + + errcode, errmsg, headers = h.getreply() + + if errcode != 200: + raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg, + headers) + + try: + sock = h._conn.sock + except AttributeError: + sock = None + + self.verbose = verbose + #print headers.dict + if headers.get('Content-encoding', '') == 'gzip': + return self.parse_response(GzipDecodedResponse(h.getfile())) + else: + return self._parse_response(h.getfile(), sock) + + transport.request = wrapped_request + + def wrap_transport(transport): + if sys.version_info[:2] == (2, 6): + wrap_request(transport) old_parse_response = transport.parse_response def wrapped_parse_response(*args, **kwargs): @@ -42,7 +98,7 @@ return old_parse_response(*args, **kwargs) except ExpatError: sys.stderr.write('Parsing error was thrown during parsing ' - 'response when provided: {}'.format(args[1])) + 'response when provided: {0}'.format(args)) raise transport.parse_response = wrapped_parse_response return transport -- To view, visit http://gerrit.ovirt.org/25201 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iebdaa04b17e2c1df1c1852ed536c5d6d8ec8d88b Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Vinzenz Feenstra <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
