jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] upload: Support continue/finish uploads ......................................................................
[FEAT] upload: Support continue/finish uploads When the upload was interrupted or didn't finished because of a warning it's possible to reuse the already uploaded data and just publish it (if it was finished) or upload the rest (when chunked mode is used). Bug: T101657 Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921 --- M pywikibot/data/api.py M pywikibot/site.py M scripts/upload.py 3 files changed, 63 insertions(+), 15 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 236f58a..bec26a6 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -113,9 +113,20 @@ """Upload failed with a warning message (passed as the argument).""" - def __init__(self, code, message): - """Constructor.""" + def __init__(self, code, message, filekey=None, offset=0): + """ + Create a new UploadWarning instance. + + @param filekey: The filekey of the uploaded file to reuse it later. If + no key is known or it is an incomplete file it may be None. + @type filekey: str or None + @param offset: The starting offset for a chunked upload. Is False when + there is no offset. + @type offset: int or bool + """ super(UploadWarning, self).__init__(code, message) + self.filekey = filekey + self.offset = offset @property def message(self): diff --git a/pywikibot/site.py b/pywikibot/site.py index 2c4f288..238c133 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -4959,7 +4959,7 @@ @deprecate_arg('imagepage', 'filepage') def upload(self, filepage, source_filename=None, source_url=None, comment=None, text=None, watch=False, ignore_warnings=False, - chunk_size=0): + chunk_size=0, _file_key=None, _offset=0): """Upload a file to the wiki. Either source_filename or source_url, but not both, must be provided. @@ -4981,6 +4981,13 @@ will only upload in chunks, if the version number is 1.20 or higher and the chunk size is positive but lower than the file size. @type chunk_size: int + @param _file_key: Reuses an already uploaded file using the filekey. If + None (default) it will upload the file. + @type _file_key: str or None + @param _offset: When file_key is not None this can be an integer to + continue a previously canceled chunked upload. If False it treats + that as a finished upload. By default starts at 0. + @type _offset: int or bool """ upload_warnings = { # map API warning codes to user error messages @@ -4996,6 +5003,8 @@ '"%(msg)s".', } + # An offset != 0 doesn't make sense without a file key + assert(_offset == 0 or _file_key is not None) # check for required user right if "upload" not in self.userinfo["rights"]: raise Error( @@ -5017,7 +5026,15 @@ token = self.tokens['edit'] result = None file_page_title = filepage.title(withNamespace=False) - if source_filename: + if _file_key and _offset is False: + pywikibot.log('Reused already upload file using ' + 'filekey "{0}"'.format(_file_key)) + # TODO: Use sessionkey instead of filekey if necessary + req = api.Request(site=self, action='upload', token=token, + filename=file_page_title, + comment=comment, text=text, + filekey=_file_key) + elif source_filename: # TODO: Dummy value to allow also Unicode names, see bug 73661 mime_filename = 'FAKE-NAME' # upload local file @@ -5032,8 +5049,10 @@ MediaWikiVersion(self.version()) >= MediaWikiVersion('1.20')) with open(source_filename, 'rb') as f: if chunked_upload: - offset = 0 - file_key = None + offset = _offset + if offset > 0: + pywikibot.log('Continuing upload from byte ' + '{0}'.format(offset)) while True: f.seek(offset) chunk = f.read(chunk_size) @@ -5045,8 +5064,8 @@ req.mime_params['chunk'] = (chunk, ("application", "octet-stream"), {'filename': mime_filename}) - if file_key: - req['filekey'] = file_key + if _file_key: + req['filekey'] = _file_key try: data = req.submit()['upload'] self._uploaddisabled = False @@ -5058,18 +5077,23 @@ if 'warnings' in data and not ignore_warnings: result = data break - file_key = data['filekey'] + _file_key = data['filekey'] throttle = False if 'offset' in data: new_offset = int(data['offset']) if offset + len(chunk) != new_offset: + pywikibot.log('Old offset: {0}; Returned ' + 'offset: {1}; Chunk size: ' + '{2}'.format(offset, new_offset, + len(chunk))) pywikibot.warning('Unexpected offset.') offset = new_offset else: pywikibot.warning('Offset was not supplied.') offset += len(chunk) if data['result'] != 'Continue': # finished - additional_parameters['filekey'] = file_key + pywikibot.log('Finished uploading last chunk.') + additional_parameters['filekey'] = _file_key break else: # not chunked upload file_contents = f.read() @@ -5113,8 +5137,19 @@ # TODO: Handle multiple warnings at the same time warning = list(result["warnings"].keys())[0] message = result["warnings"][warning] + if 'filekey' in result: + _file_key = result['filekey'] + elif 'sessionkey' in result: + # TODO: Probably needs to be reflected in the API call above + _file_key = result['sessionkey'] + pywikibot.warning('Using sessionkey instead of filekey.') + else: + _file_key = None + pywikibot.warning('No filekey defined.') raise pywikibot.UploadWarning(warning, upload_warnings[warning] - % {'msg': message}) + % {'msg': message}, + file_key=_file_key, + offset=result['offset'] if 'offset' in result else 0) elif "result" not in result: pywikibot.output(u"Upload: unrecognized response: %s" % result) if result["result"] == "Success": diff --git a/scripts/upload.py b/scripts/upload.py index 95c3a77..b7ea3fa 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -333,7 +333,7 @@ """Upload image.""" self.upload_file(self.url, debug) - def upload_file(self, file_url, debug=False): + def upload_file(self, file_url, debug=False, _file_key=None, _offset=0): """Upload the image at file_url to the target wiki. Return the filename that was used to upload the image. @@ -357,7 +357,8 @@ apiIgnoreWarnings = True if self.uploadByUrl: site.upload(imagepage, source_url=file_url, - ignore_warnings=apiIgnoreWarnings) + ignore_warnings=apiIgnoreWarnings, + _file_key=_file_key, _offset=_offset) else: if "://" in file_url: temp = self.read_file_content(file_url) @@ -365,7 +366,8 @@ temp = file_url site.upload(imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, - chunk_size=self.chunk_size) + chunk_size=self.chunk_size, + _file_key=_file_key, _offset=_offset) except pywikibot.data.api.UploadWarning as warn: pywikibot.output( @@ -380,7 +382,7 @@ if answer: self.ignoreWarning = True self.keepFilename = True - return self.upload_file(file_url, debug) + return self.upload_file(file_url, debug, warn.file_key, warn.offset) else: pywikibot.output(u"Upload aborted.") return -- To view, visit https://gerrit.wikimedia.org/r/217275 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <commodorefabia...@gmx.de> Gerrit-Reviewer: John Vandenberg <jay...@gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgr...@gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhall...@arctus.nl> Gerrit-Reviewer: XZise <commodorefabia...@gmx.de> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits