XZise has submitted this change and it was merged.

Change subject: [FIX] Support Python 3 uploads ......................................................................

[FIX] Support Python 3 uploads

This fixes bugs in uploads with Python 3 (and fixes using MIME mode with requests).

As Python 3 email does not correctly handle Content-Transfer-Encoding of 'binary', a workaround class CTEBinaryMIMEMultipart is used to strictly emit the payload unmodified.

It now also closes the file and uses open() as file() has been removed in Python 3.

Also some Pep257 fixes included.

Change-Id: I9e73d00fc59fd43a5eb6d55efdbf02dd8e605014 --- M pywikibot/data/api.py M tests/dry_api_tests.py 2 files changed, 56 insertions(+), 7 deletions(-)

Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved

diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index dd321d4..db927aa 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -9,7 +9,6 @@

from collections import MutableMapping from pywikibot.comms import http -from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart import datetime import hashlib @@ -32,11 +31,50 @@ import sys

if sys.version_info[0] > 2: + # Subclassing necessary to fix a possible bug of the email package + # in py3: see http://bugs.python.org/issue19003 + # The following solution might be removed if/once the bug is fixed, + # unless the fix is not backported to py3.x versions that should + # instead support PWB. basestring = (str, ) from urllib.parse import urlencode, unquote unicode = str + + from io import BytesIO + + import email.generator + from email.mime.multipart import MIMEMultipart as MIMEMultipartOrig + + class CTEBinaryBytesGenerator(email.generator.BytesGenerator): + + """Workaround for bug in python 3 email handling of CTE binary.""" + + def __init__(self, *args, **kwargs): + super(CTEBinaryBytesGenerator, self).__init__(*args, **kwargs) + self._writeBody = self._write_body + + def _write_body(self, msg): + if msg['content-transfer-encoding'] == 'binary': + self._fp.write(msg.get_payload(decode=True)) + else: + super(CTEBinaryBytesGenerator, self)._handle_text(msg) + + class CTEBinaryMIMEMultipart(MIMEMultipartOrig): + + """Workaround for bug in python 3 email handling of CTE binary.""" + + def as_bytes(self, unixfrom=False, policy=None): + """Return unmodified binary payload.""" + policy = self.policy if policy is None else policy + fp = BytesIO() + g = CTEBinaryBytesGenerator(fp, mangle_from_=False, policy=policy) + g.flatten(self, unixfrom=unixfrom) + return fp.getvalue() + + MIMEMultipart = CTEBinaryMIMEMultipart else: from urllib import urlencode, unquote + from email.mime.multipart import MIMEMultipart

_logger = "data.api"

@@ -437,6 +475,10 @@ content_headers.update(headers) submsg.add_header("Content-disposition", "form-data", **content_headers) + + if keytype != ("text", "plain"): + submsg['Content-Transfer-Encoding'] = 'binary' + submsg.set_payload(content) return submsg

@@ -461,12 +503,16 @@ container.attach(submsg)

# strip the headers to get the HTTP message body - body = container.as_string() - marker = "\n\n" # separates headers from body + if sys.version_info[0] > 2: + body = container.as_bytes() + marker = b"\n\n" # separates headers from body + else: + body = container.as_string() + marker = "\n\n" # separates headers from body eoh = body.find(marker) body = body[eoh + len(marker):] # retrieve the headers from the MIME object - headers = dict(list(container.items())) + headers = dict(container.items()) return headers, body

def _handle_warnings(self, result): diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py index d8622cb..c294e58 100644 --- a/tests/dry_api_tests.py +++ b/tests/dry_api_tests.py @@ -23,10 +23,14 @@ import os import sys import datetime -from email.mime.multipart import MIMEMultipart

import pywikibot -from pywikibot.data.api import Request, CachedRequest, QueryGenerator +from pywikibot.data.api import ( + Request, + MIMEMultipart, + CachedRequest, + QueryGenerator, +) from pywikibot.family import Family

from tests import _data_dir @@ -203,7 +207,6 @@ {'filename': local_filename}) self.assertEqual(file_content, submsg.get_payload(decode=True))

- @unittest.skipIf(sys.version_info[0] > 2, "Currently fails on Python 3") def test_mime_file_container(self): local_filename = os.path.join(_data_dir, 'MP_sounds.png') with open(local_filename, 'rb') as f: