#23397: Multipart base64 file decoding of fails with large files when the encoded string contains newlines. -------------------------------+-------------------------------------- Reporter: jhobbs | Owner: nobody Type: Bug | Status: new Component: HTTP handling | Version: 1.6 Severity: Normal | Resolution: Keywords: | Triage Stage: Unreviewed Has patch: 0 | Needs documentation: 0 Needs tests: 0 | Patch needs improvement: 0 Easy pickings: 0 | UI/UX: 0 -------------------------------+-------------------------------------- Description changed by jhobbs:
Old description: > Large files are files larger than the chunk size in > MultiPartParser.parse(). > > Parse tries to process base64 encoded files in chunks with lengths a > multiple of 4. The base64 encoded string can contain newline characters, > which aren't significant in base64 and should be ignored, but are counted > toward's a chunk's length in parse(). This means when whitespace is > stripped from the string, the count of base64 encoded characters may not > be a multiple of 4, leading to an "Incorrect padding" error. > > Here's a testcase for creating this failure: > > https://github.com/jhobbs/django/commit/98347e396133990b82b596310e4490369ba676d5 > > Stacktrace: > {{{ > Traceback (most recent call last): > File "/home/jason/canonical/code/django/tests/file_uploads/tests.py", > line 109, in test_big_base64_newlines_upload > "Big data" * 68000, encode=base64.encodestring) > File "/home/jason/canonical/code/django/tests/file_uploads/tests.py", > line 96, in _test_base64_upload > response = self.client.request(**r) > File "/home/jason/canonical/code/django/django/test/client.py", line > 443, in request > six.reraise(*exc_info) > File "/home/jason/canonical/code/django/django/core/handlers/base.py", > line 121, in get_response > response = wrapped_callback(request, *callback_args, > **callback_kwargs) > File "/home/jason/canonical/code/django/tests/file_uploads/views.py", > line 103, in file_upload_echo_content > r = dict((k, f.read().decode('utf-8')) for k, f in > request.FILES.items()) > File "/home/jason/canonical/code/django/django/core/handlers/wsgi.py", > line 152, in _get_files > self._load_post_and_files() > File "/home/jason/canonical/code/django/django/http/request.py", line > 249, in _load_post_and_files > self._post, self._files = self.parse_file_upload(self.META, data) > File "/home/jason/canonical/code/django/django/http/request.py", line > 214, in parse_file_upload > return parser.parse() > File > "/home/jason/canonical/code/django/django/http/multipartparser.py", line > 220, in parse > six.reraise(MultiPartParserError, MultiPartParserError(msg), > sys.exc_info()[2]) > File > "/home/jason/canonical/code/django/django/http/multipartparser.py", line > 216, in parse > chunk = base64.b64decode(chunk) > File "/usr/lib/python2.7/base64.py", line 76, in b64decode > raise TypeError(msg) > MultiPartParserError: Could not decode base64 data: > TypeError(Error('Incorrect padding',),) > }}} New description: Large files are files larger than the chunk size in MultiPartParser.parse(). parse() tries to process base64 encoded files in chunks with lengths a multiple of 4. The base64 encoded string can contain newline characters, which aren't significant in base64 and should be ignored, but are counted toward's a chunk's length in parse(). This means when whitespace is stripped from the string, the count of base64 encoded characters may not be a multiple of 4, leading to an "Incorrect padding" error. Here's a testcase for creating this failure: https://github.com/jhobbs/django/commit/98347e396133990b82b596310e4490369ba676d5 Stacktrace: {{{ Traceback (most recent call last): File "/home/jason/canonical/code/django/tests/file_uploads/tests.py", line 109, in test_big_base64_newlines_upload "Big data" * 68000, encode=base64.encodestring) File "/home/jason/canonical/code/django/tests/file_uploads/tests.py", line 96, in _test_base64_upload response = self.client.request(**r) File "/home/jason/canonical/code/django/django/test/client.py", line 443, in request six.reraise(*exc_info) File "/home/jason/canonical/code/django/django/core/handlers/base.py", line 121, in get_response response = wrapped_callback(request, *callback_args, **callback_kwargs) File "/home/jason/canonical/code/django/tests/file_uploads/views.py", line 103, in file_upload_echo_content r = dict((k, f.read().decode('utf-8')) for k, f in request.FILES.items()) File "/home/jason/canonical/code/django/django/core/handlers/wsgi.py", line 152, in _get_files self._load_post_and_files() File "/home/jason/canonical/code/django/django/http/request.py", line 249, in _load_post_and_files self._post, self._files = self.parse_file_upload(self.META, data) File "/home/jason/canonical/code/django/django/http/request.py", line 214, in parse_file_upload return parser.parse() File "/home/jason/canonical/code/django/django/http/multipartparser.py", line 220, in parse six.reraise(MultiPartParserError, MultiPartParserError(msg), sys.exc_info()[2]) File "/home/jason/canonical/code/django/django/http/multipartparser.py", line 216, in parse chunk = base64.b64decode(chunk) File "/usr/lib/python2.7/base64.py", line 76, in b64decode raise TypeError(msg) MultiPartParserError: Could not decode base64 data: TypeError(Error('Incorrect padding',),) }}} -- -- Ticket URL: <https://code.djangoproject.com/ticket/23397#comment:2> Django <https://code.djangoproject.com/> The Web framework for perfectionists with deadlines. -- You received this message because you are subscribed to the Google Groups "Django updates" group. To unsubscribe from this group and stop receiving emails from it, send an email to django-updates+unsubscr...@googlegroups.com. To post to this group, send email to django-updates@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/django-updates/064.47bbc94420f505a4c0ee9e5bf161e675%40djangoproject.com. For more options, visit https://groups.google.com/d/optout.