Pywikibot-commits August 2015

pywikibot-commits@lists.wikimedia.org

2 participants
240 discussions

[Gerrit] Add fallback for v1.14 siteinfo 'fileextensions' - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Add fallback for v1.14 siteinfo 'fileextensions' ...................................................................... Add fallback for v1.14 siteinfo 'fileextensions' Change-Id: I97892edc04f9548aba15b69206a7b8ecaa98a901 --- M pywikibot/site.py M tests/site_tests.py M tests/utils.py 3 files changed, 33 insertions(+), 12 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 695359c..be830ba 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -1270,6 +1270,9 @@ 'levels': ['', 'autoconfirmed', 'sysop'], 'types': ['create', 'edit', 'move', 'upload'] } + elif key == 'fileextensions': + # the default file extensions in MediaWiki + return [{'ext': ext} for ext in ['png', 'gif', 'jpg', 'jpeg']] else: return pywikibot.tools.EMPTY_DEFAULT diff --git a/tests/site_tests.py b/tests/site_tests.py index 0b9c726..84fdf31 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -33,7 +33,7 @@ DefaultWikidataClientTestCase, AlteredDefaultSiteTestCase, ) -from tests.utils import allowed_failure, allowed_failure_if +from tests.utils import allowed_failure, allowed_failure_if, entered_loop from tests.basepage_tests import BasePageLoadRevisionsCachingTestBase if sys.version_info[0] > 2: @@ -1765,7 +1765,7 @@ self.assertEqual(len(mypage._revisions), 12) -class TestSiteInfo(WikimediaDefaultSiteTestCase): +class TestSiteInfo(DefaultSiteTestCase): """Test cases for Site metadata and capabilities.""" @@ -1788,20 +1788,31 @@ self.assertEqual(mysite.case(), mysite.siteinfo['case']) self.assertEqual(re.findall("\$1", mysite.siteinfo['articlepath']), ["$1"]) - def entered_loop(iterable): - for iterable_item in iterable: - return True - return False - + def test_properties_with_defaults(self): + """Test the siteinfo properties with defaults.""" + # This does not test that the defaults work correct, + # unless the default site is a version needing these defaults + # 'fileextensions' introduced in v1.14: + self.assertIsInstance(self.site.siteinfo.get('fileextensions'), list) + self.assertIn('fileextensions', self.site.siteinfo) + self.assertIn({'ext': 'png'}, self.site.siteinfo['fileextensions']) + # 'restrictions' introduced in v1.23: + mysite = self.site self.assertIsInstance(mysite.siteinfo.get('restrictions'), dict) self.assertIn('restrictions', mysite.siteinfo) - # the following line only works in 1.23+ - self.assertTrue(mysite.siteinfo.is_recognised('restrictions')) - del mysite.siteinfo._cache['restrictions'] - self.assertIsInstance(mysite.siteinfo.get('restrictions', cache=False), dict) - self.assertNotIn('restrictions', mysite.siteinfo) + self.assertIn('cascadinglevels', self.site.siteinfo['restrictions']) + def test_no_cache(self): + """Test siteinfo caching can be disabled.""" + if 'fileextensions' in self.site.siteinfo._cache: + del self.site.siteinfo._cache['fileextensions'] + self.site.siteinfo.get('fileextensions', cache=False) + self.assertNotIn('fileextensions', self.site.siteinfo) + + def test_not_exists(self): + """Test accessing a property not in siteinfo.""" not_exists = 'this-property-does-not-exist' + mysite = self.site self.assertRaises(KeyError, mysite.siteinfo.__getitem__, not_exists) self.assertNotIn(not_exists, mysite.siteinfo) self.assertEqual(len(mysite.siteinfo.get(not_exists)), 0) diff --git a/tests/utils.py b/tests/utils.py index ae5face..c882d61 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -119,6 +119,13 @@ return gen +def entered_loop(iterable): + """Return True if iterable contains items.""" + for iterable_item in iterable: + return True + return False + + class WarningSourceSkipContextManager(warnings.catch_warnings): """ -- To view, visit https://gerrit.wikimedia.org/r/230486 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I97892edc04f9548aba15b69206a7b8ecaa98a901 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] [FIX] Win32 UI: Ignore fileno if not supported - change (pywikibot/compat)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Win32 UI: Ignore fileno if not supported ...................................................................... [FIX] Win32 UI: Ignore fileno if not supported On some environments, like IDLE, getting the fileno of a std stream raises an UnsupportedOperation error. Backport from core dd808ca6e0a5eaa25c10316041aa2f975a94df5a Change-Id: I1557fe83ffb06c2702042ab4d68af04a0398781b Signed-off-by: xqt <info(a)gno.de> Bug: T99368 --- M userinterfaces/win32_unicode.py 1 file changed, 68 insertions(+), 56 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/userinterfaces/win32_unicode.py b/userinterfaces/win32_unicode.py index ae00ef6..65027b2 100644 --- a/userinterfaces/win32_unicode.py +++ b/userinterfaces/win32_unicode.py @@ -1,25 +1,25 @@ -# Stdout, stderr and argv support: +"""Stdout, stderr and argv support for unicode.""" ############################################## # Support for unicode in windows cmd.exe -# Posted on Stack Overflow [1], available under CC-BY-SA [2] -# +# Posted on Stack Overflow [1], available under CC-BY-SA 3.0 [2] +# # Question: "Windows cmd encoding change causes Python crash" [3] by Alex [4], # Answered [5] by David-Sarah Hopwood [6]. # -# [1] http://stackoverflow.com -# [2] http://creativecommons.org/licenses/by-sa/3.0/ -# [3] http://stackoverflow.com/questions/878972 -# [4] http://stackoverflow.com/users/85185 -# [4] http://stackoverflow.com/a/3259271/118671 -# [5] http://stackoverflow.com/users/393146 +# [1] https://stackoverflow.com +# [2] https://creativecommons.org/licenses/by-sa/3.0/ +# [3] https://stackoverflow.com/questions/878972 +# [4] https://stackoverflow.com/users/85185 +# [4] https://stackoverflow.com/a/3259271/118671 +# [5] https://stackoverflow.com/users/393146 # ################################################ # -# stdin support added by Merlijn van Deen <valhallasw(a)gmail.com>, march 2012 -# Licensed under both CC-BY-SA as the MIT license. +# stdin support added by Merlijn van Deen <valhallasw(a)gmail.com>, March 2012 +# Licensed under both CC-BY-SA and the MIT license. # ################################################ - +from io import UnsupportedOperation import sys stdin = sys.stdin stdout = sys.stdout @@ -27,8 +27,8 @@ argv = sys.argv if sys.platform == "win32": import codecs - from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int, \ - create_unicode_buffer + from ctypes import WINFUNCTYPE, windll, POINTER + from ctypes import byref, c_int, create_unicode_buffer from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR try: from ctypes.wintypes import LPVOID @@ -51,50 +51,53 @@ # This also fixes <http://bugs.python.org/issue1602>. # Credit to Michael Kaplan <http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx> # and TZOmegaTZIOY - # <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-cause…>. + # <https://stackoverflow.com/questions/878972/windows-cmd-encoding-change-caus…>. try: - # <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx> + # <https://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx> # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); # returns INVALID_HANDLE_VALUE, NULL, or a valid handle # - # <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx> + # <https://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx> # DWORD WINAPI GetFileType(DWORD hFile); # - # <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx> + # <https://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx> # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32)) STD_INPUT_HANDLE = DWORD(-10) STD_OUTPUT_HANDLE = DWORD(-11) - STD_ERROR_HANDLE = DWORD(-12) + STD_ERROR_HANDLE = DWORD(-12) GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32)) - FILE_TYPE_CHAR = 0x0002 + FILE_TYPE_CHAR = 0x0002 FILE_TYPE_REMOTE = 0x8000 - GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD)) \ - (("GetConsoleMode", windll.kernel32)) + GetConsoleMode = (WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD)) + (('GetConsoleMode', windll.kernel32))) INVALID_HANDLE_VALUE = DWORD(-1).value def not_a_console(handle): if handle == INVALID_HANDLE_VALUE or handle is None: return True - return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR - or GetConsoleMode(handle, byref(DWORD())) == 0) + return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or + GetConsoleMode(handle, byref(DWORD())) == 0) - old_stdin_fileno = None - old_stdout_fileno = None - old_stderr_fileno = None + def old_fileno(std_name): + # some environments like IDLE don't support the fileno operation + # handle those like std streams which don't have fileno at all + std = getattr(sys, 'std{0}'.format(std_name)) + if hasattr(std, 'fileno'): + try: + return std.fileno() + except UnsupportedOperation: + pass - if hasattr(sys.stdin, 'fileno'): - old_stdin_fileno = sys.stdin.fileno() - if hasattr(sys.stdout, 'fileno'): - old_stdout_fileno = sys.stdout.fileno() - if hasattr(sys.stderr, 'fileno'): - old_stderr_fileno = sys.stderr.fileno() + old_stdin_fileno = old_fileno('in') + old_stdout_fileno = old_fileno('out') + old_stderr_fileno = old_fileno('err') - STDIN_FILENO = 0 + STDIN_FILENO = 0 STDOUT_FILENO = 1 STDERR_FILENO = 2 - real_stdin = (old_stdin_fileno == STDIN_FILENO) + real_stdin = (old_stdin_fileno == STDIN_FILENO) real_stdout = (old_stdout_fileno == STDOUT_FILENO) real_stderr = (old_stderr_fileno == STDERR_FILENO) @@ -102,7 +105,7 @@ hStdin = GetStdHandle(STD_INPUT_HANDLE) if not_a_console(hStdin): real_stdin = False - + if real_stdout: hStdout = GetStdHandle(STD_OUTPUT_HANDLE) if not_a_console(hStdout): @@ -114,34 +117,40 @@ real_stderr = False if real_stdin: - ReadConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPVOID, DWORD, POINTER(DWORD), \ + ReadConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPVOID, DWORD, POINTER(DWORD), LPVOID)(("ReadConsoleW", windll.kernel32)) - + class UnicodeInput: + + """Unicode terminal input class.""" + def __init__(self, hConsole, name, bufsize=1024): self._hConsole = hConsole self.bufsize = bufsize self.buffer = create_unicode_buffer(bufsize) self.name = name self.encoding = 'utf-8' - + def readline(self): - maxnum = DWORD(self.bufsize-1) + maxnum = DWORD(self.bufsize - 1) numrecv = DWORD(0) result = ReadConsoleW(self._hConsole, self.buffer, maxnum, byref(numrecv), None) if not result: raise Exception("stdin failure") return self.buffer.value[:numrecv.value].encode(self.encoding) - + if real_stdout or real_stderr: # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, # LPDWORD lpCharsWritten, LPVOID lpReserved); - WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), \ + WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)(("WriteConsoleW", windll.kernel32)) class UnicodeOutput: + + """Unicode terminal output class.""" + def __init__(self, hConsole, stream, fileno, name): self._hConsole = hConsole self._stream = stream @@ -155,16 +164,19 @@ def isatty(self): return False + def close(self): # don't really close the handle, that would only cause problems self.closed = True + def fileno(self): return self._fileno + def flush(self): if self._hConsole is None: try: self._stream.flush() - except Exception, e: + except Exception as e: _complain("%s.flush: %r from %r" % (self.name, e, self._stream)) raise @@ -183,7 +195,7 @@ n = DWORD(0) # There is a shorter-than-documented limitation on the # length of the string passed to WriteConsoleW (see - # <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. + # <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) @@ -191,9 +203,10 @@ raise IOError("WriteConsoleW returned %r, n.value = %r" % (retval, n.value)) remaining -= n.value - if remaining == 0: break + if remaining == 0: + break text = text[n.value:] - except Exception, e: + except Exception as e: _complain("%s.write: %r" % (self.name, e)) raise @@ -201,36 +214,35 @@ try: for line in lines: self.write(line) - except Exception, e: + except Exception as e: _complain("%s.writelines: %r" % (self.name, e)) raise if real_stdin: - stdin = UnicodeInput(hStdin, name='<Unicode console stdin>') + stdin = UnicodeInput(hStdin, name='<Unicode console stdin>') if real_stdout: stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, - '<Unicode console stdout>') + '<Unicode console stdout>') else: stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, - '<Unicode redirected stdout>') + '<Unicode redirected stdout>') if real_stderr: stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, - '<Unicode console stderr>') + '<Unicode console stderr>') else: stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, - '<Unicode redirected stderr>') - except Exception, e: + '<Unicode redirected stderr>') + except Exception as e: _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,)) - # While we're at it, let's unmangle the command-line arguments: # This works around <http://bugs.python.org/issue2128>. GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) - CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int)) \ - (("CommandLineToArgvW", windll.shell32)) + CommandLineToArgvW = (WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int)) + (('CommandLineToArgvW', windll.shell32))) argc = c_int(0) argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) -- To view, visit https://gerrit.wikimedia.org/r/230529 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1557fe83ffb06c2702042ab4d68af04a0398781b Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] [FIX] Remove crossnamespaces out of the scope - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Remove crossnamespaces out of the scope ...................................................................... [FIX] Remove crossnamespaces out of the scope crossnamespace is used on Wikipedia and Wikisource projects only. interwiki.py is the only script which uses it. But Wikipedia and Wikisource projects have wikibase support and don't need it anymore. Bug: T75170 Change-Id: I931e0ff66069d172107d4f91c6bff3bb6fffaba6 --- M pywikibot/families/wikipedia_family.py M pywikibot/families/wikisource_family.py 2 files changed, 0 insertions(+), 159 deletions(-) Approvals: XZise: Looks good to me, but someone else must approve Ladsgroup: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/families/wikipedia_family.py b/pywikibot/families/wikipedia_family.py index 1caf63c..5190f33 100644 --- a/pywikibot/families/wikipedia_family.py +++ b/pywikibot/families/wikipedia_family.py @@ -365,161 +365,6 @@ 'nds', 'co', 'mi', 'mr', 'id', 'lv', 'sw', 'tt', 'uk', 'vo', 'ga', 'na', 'es', 'nl', 'da', 'dk', 'sv', 'test'] - self.crossnamespace[0] = { - '_default': { - 'pt': [102], - 'als': [104], - 'ar': [104], - 'de': [4], - 'en': [12], - 'es': [104], - 'fi': [4], - 'fr': [104], - 'hr': [102], - 'lt': [104], - }, - 'km': { - '_default': [0, 4, 12], - }, - # wrong wikipedia namespace alias - 'mzn': { - '_default': [0, 4], - - }, - } - self.crossnamespace[1] = { - '_default': { - 'pt': [103], - 'als': [105], - 'ar': [105], - 'en': [13], - 'es': [105], - 'fi': [5], - 'fr': [105], - 'hr': [103], - 'lt': [105], - }, - } - self.crossnamespace[4] = { - '_default': { - '_default': [12], - }, - 'de': { - '_default': [0, 10, 12], - 'el': [100, 12], - 'es': [104, 12], - }, - 'fi': { - '_default': [0, 12] - }, - 'mzn': { - '_default': [0, 12] - }, - } - self.crossnamespace[5] = { - 'fi': { - '_default': [1]} - } - self.crossnamespace[12] = { - '_default': { - '_default': [4], - }, - 'en': { - '_default': [0, 4], - }, - } - self.crossnamespace[13] = { - 'en': { - '_default': [0], - }, - } - self.crossnamespace[102] = { - 'pt': { - '_default': [0], - 'als': [0, 104], - 'ar': [0, 104], - 'es': [0, 104], - 'fr': [0, 104], - 'lt': [0, 104] - }, - 'hr': { - '_default': [0], - 'als': [0, 104], - 'ar': [0, 104], - 'es': [0, 104], - 'fr': [0, 104], - 'lt': [0, 104] - }, - } - self.crossnamespace[103] = { - 'pt': { - '_default': [1], - 'als': [1, 105], - 'es': [1, 105], - 'fr': [1, 105], - 'lt': [1, 105] - }, - 'hr': { - '_default': [1], - 'als': [1, 105], - 'es': [1, 105], - 'fr': [1, 105], - 'lt': [1, 105] - }, - } - self.crossnamespace[104] = { - 'als': { - '_default': [0], - 'pt': [0, 102], - 'hr': [0, 102], - }, - 'ar': { - '_default': [0, 100], - 'hr': [0, 102], - 'pt': [0, 102], - }, - 'es': { - '_default': [0], - 'pt': [0, 102], - 'hr': [0, 102], - }, - 'fr': { - '_default': [0], - 'pt': [0, 102], - 'hr': [0, 102], - }, - 'lt': { - '_default': [0], - 'pt': [0, 102], - 'hr': [0, 102], - }, - } - self.crossnamespace[105] = { - 'als': { - '_default': [1], - 'pt': [0, 103], - 'hr': [0, 103], - }, - 'ar': { - '_default': [1, 101], - }, - 'es': { - '_default': [1], - 'pt': [0, 103], - 'hr': [0, 103], - }, - 'fr': { - '_default': [1], - 'pt': [0, 103], - 'hr': [0, 103], - }, - 'lt': { - '_default': [1], - 'pt': [0, 103], - 'hr': [0, 103], - }, - } - def get_known_families(self, site): """Override the family interwiki prefixes for each site.""" # In Swedish Wikipedia 's:' is part of page title not a family diff --git a/pywikibot/families/wikisource_family.py b/pywikibot/families/wikisource_family.py index 8e47739..dc55fa8 100644 --- a/pywikibot/families/wikisource_family.py +++ b/pywikibot/families/wikisource_family.py @@ -92,10 +92,6 @@ 'zh': [102], } - for key, values in self.authornamespaces.items(): - for item in values: - self.crossnamespace[item].update({key: self.authornamespaces}) - # Subpages for documentation. # TODO: List is incomplete, to be completed for missing languages. # TODO: Remove comments for appropriate pages -- To view, visit https://gerrit.wikimedia.org/r/230377 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I931e0ff66069d172107d4f91c6bff3bb6fffaba6 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Workaround Python 2.6.5 issue #6906 - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Workaround Python 2.6.5 issue #6906 ...................................................................... Workaround Python 2.6.5 issue #6906 Before Python 2.6.6 Tk set several environment variables using unicode instead of str, which fails on Win32. Update tests.utils.execute to encode these environment variables, and improve the generated TypeError message to include original message. Bug: T108035 Change-Id: I275323c9a0af5d32d023615e5455efedcf6b6496 --- M tests/utils.py 1 file changed, 13 insertions(+), 5 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/utils.py b/tests/utils.py index ecf02c2..ae5face 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,6 +33,7 @@ from pywikibot.site import Namespace from pywikibot.data.api import CachedRequest from pywikibot.data.api import Request as _original_Request +from pywikibot.tools import PYTHON_VERSION from tests import _pwb_py from tests import unittest # noqa @@ -525,6 +526,12 @@ # str() on Python 2. env = os.environ.copy() + # Python issue 6906 + if PYTHON_VERSION < (2, 6, 6): + for var in ('TK_LIBRARY', 'TCL_LIBRARY', 'TIX_LIBRARY'): + if var in env: + env[var] = env[var].encode('mbcs') + # Prevent output by test package; e.g. 'max_retries reduced from x to y' env[str('PYWIKIBOT_TEST_QUIET')] = str('1') @@ -551,21 +558,22 @@ try: p = subprocess.Popen(command, env=env, **options) - except TypeError: + except TypeError as e: # Generate a more informative error if sys.platform == 'win32' and sys.version_info[0] < 3: unicode_env = [(k, v) for k, v in os.environ.items() if not isinstance(k, str) or not isinstance(v, str)] if unicode_env: - raise TypeError('os.environ must contain only str: %r' - % unicode_env) + raise TypeError( + '%s: unicode in os.environ: %r' % (e, unicode_env)) + child_unicode_env = [(k, v) for k, v in env.items() if not isinstance(k, str) or not isinstance(v, str)] if child_unicode_env: - raise TypeError('os.environ must contain only str: %r' - % child_unicode_env) + raise TypeError( + '%s: unicode in child env: %r' % (e, child_unicode_env)) raise if data_in is not None: -- To view, visit https://gerrit.wikimedia.org/r/229400 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I275323c9a0af5d32d023615e5455efedcf6b6496 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Add -unconnectedpages to pagegenerators - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Add -unconnectedpages to pagegenerators ...................................................................... Add -unconnectedpages to pagegenerators Since now Wikibase client does support API wrapper around Special:UnconnectedPages (commits: 6f037b08, ff02f539, bd4d1e07 in mediawiki/extensions/Wikibase) and it's deployed, it would be really good to have such a generator. Bug: T56738 Change-Id: I6b91bd4cf6ceaf5cd71889a81a639a88a8e8d117 --- M pywikibot/pagegenerators.py M pywikibot/site.py 2 files changed, 42 insertions(+), 1 deletion(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 7971750..2205c81 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -156,6 +156,10 @@ given as -recentchanges:x, will work on the x most recently changed pages. +-unconnectedpages Work on the most recent unconnected pages to the Wikibase + repository. Given as -unconnectedpages:x, will work on the + x most recent unconnected pages. + -ref Work on all pages that link to a certain page. Argument can also be given as "-ref:referredpagetitle". @@ -690,6 +694,12 @@ gen = NewpagesPageGenerator(namespaces=namespaces, total=total, site=self.site) + elif arg.startswith('-unconnectedpages'): + namespaces = self.namespaces or 0 + total = 60 + if len(arg) >= 18: + total = int(arg[18:]) + gen = UnconnectedPageGenerator(total=total, site=self.site) elif arg.startswith('-imagesused'): imagelinkstitle = arg[len('-imagesused:'):] if not imagelinkstitle: @@ -931,7 +941,7 @@ @param step: Maximum number of pages to retrieve per API query @type step: int - @param total: Maxmum number of pages to retrieve in total + @param total: Maxmium number of pages to retrieve in total @type total: int @param site: Site for generator results. @type site: L{pywikibot.site.BaseSite} @@ -1016,6 +1026,25 @@ return gen +def UnconnectedPageGenerator(site=None, step=None, total=None): + """ + Iterate Page objects for all unconnected pages to a Wikibase repository. + + @param step: Maximum number of pages to retrieve per API query + @type step: int + @param total: Maximum number of pages to retrieve in total + @type total: int + @param site: Site for generator results. + @type site: L{pywikibot.site.APISite} + """ + if site is None: + site = pywikibot.Site() + if not site.data_repository(): + raise ValueError('The given site does not have Wikibase repository.') + for page in site.unconnected_pages(step=step, total=total): + yield page + + @deprecated_args(referredImagePage='referredFilePage') def FileLinksGenerator(referredFilePage, step=None, total=None, content=False): """Yield Pages on which the file referredFilePage is displayed.""" diff --git a/pywikibot/site.py b/pywikibot/site.py index e77315c..bc90ac2 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -5692,6 +5692,18 @@ step=step, total=total) return lrgen + def unconnected_pages(self, step=None, total=None): + """Yield Page objects from Special:UnconnectedPages. + + @param step: request batch size + @param total: number of pages to return + """ + upgen = self._generator(api.PageGenerator, + type_arg='querypage', + gqppage='UnconnectedPages', + step=step, total=total) + return upgen + @deprecated_args(lvl='level') def protectedpages(self, namespace=0, type='edit', level=False, total=None): """ -- To view, visit https://gerrit.wikimedia.org/r/228575 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6b91bd4cf6ceaf5cd71889a81a639a88a8e8d117 Gerrit-PatchSet: 5 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] [IMPROV] pagegenerators: Use standard deprecation - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [IMPROV] pagegenerators: Use standard deprecation ...................................................................... [IMPROV] pagegenerators: Use standard deprecation The deprecation message in pagegenerators when the old -logevents syntax is used is now using the standard text. This is also fixing a small syntax error as the colon between the argument name and value was missing in the message. Change-Id: I6b52748d29e866e7598d0d0709eb27043229d2c5 --- M pywikibot/pagegenerators.py M pywikibot/tools/__init__.py M tests/pagegenerators_tests.py 3 files changed, 16 insertions(+), 11 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 7971750..77bb77e 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -39,6 +39,7 @@ deprecated, deprecated_args, redirect_func, + issue_deprecation_warning, DequeGenerator, intersect_generators, filter_unique, @@ -793,10 +794,11 @@ user = None else: user = None - warn(u'The usage of "{0}" is deprecated. Use -logevents ' - u'"{1}" instead'.format( - arg, ','.join((mode, user or '', str(total)))), - ArgumentDeprecationWarning, 2) + issue_deprecation_warning( + 'The usage of "{0}"'.format(arg), + '-logevents:"{0}"'.format( + ','.join((mode, user or '', str(total)))), + 2, ArgumentDeprecationWarning) gen = self._parse_log_events(mode, user, total) if gen: diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py index 34be4cc..49db6c6 100644 --- a/pywikibot/tools/__init__.py +++ b/pywikibot/tools/__init__.py @@ -1095,14 +1095,17 @@ return outer_wrapper -def issue_deprecation_warning(name, instead, depth): +def issue_deprecation_warning(name, instead, depth, warning_class=None): """Issue a deprecation warning.""" if instead: - warn('{0} is deprecated; use {1} instead.'.format(name, instead), - DeprecationWarning, depth + 1) + if warning_class is None: + warning_class = DeprecationWarning + warn(u'{0} is deprecated; use {1} instead.'.format(name, instead), + warning_class, depth + 1) else: - warn(u'{0} is deprecated.'.format(name), _NotImplementedWarning, - depth + 1) + if warning_class is None: + warning_class = _NotImplementedWarning + warn('{0} is deprecated.'.format(name), warning_class, depth + 1) @add_full_name diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index dc55725..dccd20c 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -783,8 +783,8 @@ def test_logevents_default(self): gf = pagegenerators.GeneratorFactory(site=self.site) self.assertTrue(gf.handleArg('-newuserslog')) - self.assertDeprecation('The usage of "-newuserslog" is deprecated.' - ' Use -logevents "newusers,,500" instead') + self.assertOneDeprecationParts('The usage of "-newuserslog"', + '-logevents:"newusers,,500"') gen = gf.getCombinedGenerator() self.assertIsNotNone(gen) pages = set(gen) -- To view, visit https://gerrit.wikimedia.org/r/228582 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6b52748d29e866e7598d0d0709eb27043229d2c5 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Let redirect.py fix item redirect in Wikidata - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Let redirect.py fix item redirect in Wikidata ...................................................................... Let redirect.py fix item redirect in Wikidata Bug: T77025 Change-Id: Ic867daa1d38973a26e1ff3ff0a3d00c77aed7e41 --- M scripts/redirect.py 1 file changed, 8 insertions(+), 0 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/redirect.py b/scripts/redirect.py index 93f2b55..963c3e9 100755 --- a/scripts/redirect.py +++ b/scripts/redirect.py @@ -387,6 +387,8 @@ }) super(RedirectRobot, self).__init__(**kwargs) self.site = pywikibot.Site() + self.repo = self.site.data_repository() + self.is_repo = self.repo if self.repo == self.site else None self.action = action self.generator = generator self.exiting = False @@ -631,6 +633,12 @@ except pywikibot.BadTitle: pywikibot.output(u"Bad Title Error") break + if self.is_repo and redir.namespace() == self.repo.item_namespace: + redir = pywikibot.ItemPage(self.repo, redir.title()) + targetPage = pywikibot.ItemPage(self.repo, targetPage.title()) + pywikibot.output('Fixing double item redirect') + redir.set_redirect_target(targetPage) + break redir.set_redirect_target(targetPage, keep_section=True, save=False) summary = i18n.twtranslate(self.site, 'redirect-fix-double', {'to': targetPage.title(asLink=True)} -- To view, visit https://gerrit.wikimedia.org/r/227992 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic867daa1d38973a26e1ff3ff0a3d00c77aed7e41 Gerrit-PatchSet: 3 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Add site detection tests - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Add site detection tests ...................................................................... Add site detection tests Bug: T88601 Change-Id: I73506739c68c42c1150270c991ad1217cf7d3793 --- M tests/__init__.py A tests/site_detect_tests.py M tox.ini 3 files changed, 177 insertions(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/__init__.py b/tests/__init__.py index 82757ed..c5a7d42 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -101,7 +101,10 @@ disabled_tests = { 'textlib': [ 'test_interwiki_format', # example; very slow test - ] + ], + 'site_detect': [ + 'test_IWM', # very slow and tests include unnecessary sites + ], } diff --git a/tests/site_detect_tests.py b/tests/site_detect_tests.py new file mode 100644 index 0000000..a914a3b --- /dev/null +++ b/tests/site_detect_tests.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +"""Test for site detection.""" +# +# (C) Pywikibot team, 2014-2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import unicode_literals + +__version__ = '$Id$' + +from pywikibot.site_detect import MWSite +from pywikibot.tools import PY2 + +from tests.aspects import unittest, TestCase + +if not PY2: + basestring = (str,) + + +class TestWikiSiteDetection(TestCase): + + """Test Case for MediaWiki detection and site object creation.""" + + family = 'meta' + code = 'meta' + net = True + + def setUp(self): + """Set up test.""" + self.failures = {} + self.errors = {} + self.pass_no = 0 + self.total = 0 + super(TestWikiSiteDetection, self).setUp() + + def tearDown(self): + """Tear Down test.""" + super(TestWikiSiteDetection, self).tearDown() + print('Out of %d sites, %d tests passed, %d tests failed ' + 'and %d tests raised an error' + % (self.total, self.pass_no, len(self.failures), len(self.errors) + ) + ) + if self.failures or self.errors: + print('Failures:\n' + '\n'.join('%s : %s' + % (key, value) for (key, value) in self.failures.items())) + print('Errors:\n' + '\n'.join('%s : %s' + % (key, value) for (key, value) in self.errors.items())) + + def _wiki_detection(self, url, result): + """Perform one load test.""" + self.total += 1 + try: + site = MWSite(url) + except Exception as e: + print('failed on ' + url) + self.errors[url] = e + return + try: + if result is None: + self.assertIsNone(site) + else: + self.assertIsInstance(site, result) + self.pass_no += 1 + except AssertionError as error: + self.failures[url] = error + + def assertSite(self, url): + """Assert a MediaWiki site can be loaded from the url.""" + self._wiki_detection(url, MWSite) + + def assertNoSite(self, url): + """Assert a url is not a MediaWiki site.""" + self._wiki_detection(url, None) + + def test_IWM(self): + """Test the load_site method for MW sites on the IWM list.""" + data = self.get_site().siteinfo['interwikimap'] + for item in data: + if 'local' not in item: + self.total += 1 + url = item['url'] + try: + site = MWSite(url) + except Exception as error: + print('failed to load ' + url) + self.errors[url] = error + continue + if type(site) is MWSite: + try: + version = site.version + except Exception as error: + print('failed to get version of ' + url) + self.errors[url] = error + else: + try: + self.assertIsInstance(version, basestring) + self.assertRegex(version, r'^\d\.\d+.*') + self.pass_no += 1 + except AssertionError as error: + print('failed to parse version of ' + url) + self.failures[url] = error + + def test_detect_site(self): + """Test detection of MediaWiki sites.""" + self.assertSite('http://botwiki.sno.cc/wiki/$1') + self.assertSite('http://glossary.reuters.com/index.php?title=$1') + self.assertSite('http://www.livepedia.gr/index.php?title=$1') + self.assertSite('http://guildwars.wikia.com/wiki/$1') + self.assertSite('http://www.gutenberg.org/wiki/$1') + self.assertSite('http://www.hrwiki.org/index.php/$1') + self.assertSite('http://wiki.rennkuckuck.de/index.php/$1') + self.assertSite('http://www.proofwiki.org/wiki/$1') + self.assertSite('http://wiki.rennkuckuck.de/index.php/$1') + self.assertSite('http://www.gutenberg.org/wiki/$1') + self.assertSite( + 'http://www.ck-wissen.de/ckwiki/index.php?title=$1') + self.assertSite('http://en.citizendium.org/wiki/$1') + self.assertSite( + 'http://www.lojban.org/tiki/tiki-index.php?page=$1') + self.assertSite('http://www.EcoReality.org/wiki/$1') + self.assertSite('http://www.wikichristian.org/index.php?title=$1') + self.assertSite('http://wikitree.org/index.php?title=$1') + self.assertEqual(self.pass_no, 16) + self.assertEqual(len(self.failures), 0) + self.assertEqual(len(self.errors), 0) + + def test_detect_failure(self): + """Test detection failure for MediaWiki sites with an API.""" + self.assertNoSite('https://en.wikifur.com/wiki/$1') + # api.php is not available + self.assertNoSite('http://wiki.animutationportal.com/index.php/$1') + # API is disabled + self.assertNoSite('http://wiki.linuxquestions.org/wiki/$1') + # offline + self.assertNoSite('http://seattlewiki.org/wiki/$1') + self.assertEqual(len(self.errors), 4) + + def test_pre_api_sites(self): + """Test detection of MediaWiki sites prior to the API.""" + self.assertNoSite('http://www.wikif1.org/$1') + self.assertNoSite('http://www.thelemapedia.org/index.php/$1') + self.assertNoSite('http://esperanto.blahus.cz/cxej/vikio/index.php/$1') + self.assertNoSite('http://www.werelate.org/wiki/$1') + self.assertNoSite('http://www.otterstedt.de/wiki/index.php/$1') + self.assertNoSite('http://kb.mozillazine.org/$1') + self.assertEqual(len(self.errors), 6) + + def test_detect_nosite(self): + """Test detection of non-wiki sites.""" + self.assertNoSite('http://bluwiki.com/go/$1') + self.assertNoSite('http://www.imdb.com/name/nm$1/') + self.assertNoSite('http://www.ecyrd.com/JSPWiki/Wiki.jsp?page=$1') + self.assertNoSite('http://operawiki.info/$1') + self.assertNoSite( + 'http://www.tvtropes.org/pmwiki/pmwiki.php/Main/$1') + self.assertNoSite('http://c2.com/cgi/wiki?$1') + self.assertNoSite('https://phabricator.wikimedia.org/$1') + self.assertNoSite( + 'http://www.merriam-webster.com/cgi-bin/dictionary?book=Dictionary&va=$1') + self.assertNoSite('http://arxiv.org/abs/$1') + self.assertNoSite('http://musicbrainz.org/doc/$1') + self.assertNoSite('http://wiki.animutationportal.com/index.php/$1') + self.assertEqual(len(self.errors), 11) + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass diff --git a/tox.ini b/tox.ini index 5679523..c85de9f 100644 --- a/tox.ini +++ b/tox.ini @@ -143,6 +143,7 @@ tests/pwb/ \ tests/pwb_tests.py \ tests/script_tests.py \ + tests/site_detect_tests.py \ tests/tests_tests.py \ tests/timestripper_tests.py \ tests/thread_tests.py \ -- To view, visit https://gerrit.wikimedia.org/r/220452 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I73506739c68c42c1150270c991ad1217cf7d3793 Gerrit-PatchSet: 8 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Maverick <manpreetkaur9411(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: VcamX <vcamx3(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Use unicodedata2>=7.0.0-2 - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Use unicodedata2>=7.0.0-2 ...................................................................... Use unicodedata2>=7.0.0-2 Bug: T108068 Change-Id: Icf04bb53ee4412d7142c5ccaafc1e5092aa2b9b4 --- M requirements.txt M setup.py 2 files changed, 2 insertions(+), 4 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/requirements.txt b/requirements.txt index 629ede0..e12c55c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ ipaddr>=2.1.10 ; python_version < '3' -git+https://github.com/jayvdb/unicodedata2@issue_2#egg=unicodedata2-7.0.0 ; python_version < '3' +unicodedata2>=7.0.0-2 ; python_version < '3' # core interwiki_graph.py: git+https://github.com/nlhepler/pydot#egg=pydot-1.0.29 diff --git a/setup.py b/setup.py index a956d07..eab2bcf 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ # 0.6.1 supports socket.io 1.0, but WMF is using 0.9 (T91393 and T85716) 'rcstream': ['socketIO-client<0.6.1'], 'security': ['requests[security]'], - 'unicode7': ['unicodedata2'] if PY2 else [], } if PY2: @@ -59,6 +58,7 @@ extra_deps.update({ 'csv': ['unicodecsv'], 'MySQL': ['oursql'], + 'unicode7': ['unicodedata2>=7.0.0-2'], }) script_deps = { @@ -109,8 +109,6 @@ script_deps['replicate_wiki.py'] = ['argparse'] dependencies.append('future>=0.15.0') # provides collections backports - dependency_links.append( - 'git+https://github.com/jayvdb/unicodedata2@issue_2#egg=unicodedata2-7.0.0') dependencies += extra_deps['unicode7'] # T102461 workaround -- To view, visit https://gerrit.wikimedia.org/r/230026 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icf04bb53ee4412d7142c5ccaafc1e5092aa2b9b4 Gerrit-PatchSet: 5 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

[Gerrit] Load Flow object content in Pywikibot - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Load Flow object content in Pywikibot ...................................................................... Load Flow object content in Pywikibot This change allows Pywikibot bots to load basic Flow content from Flow-enabled wikis. This includes loading the contents of posts, loading topics from boards, and loading replies to topics and posts. Tests have been added to ensure that content is indeed loaded. Bug: T101260 Bug: T101261 Bug: T101262 Bug: T101263 Change-Id: Ibd84d0763a7745da6410c194b8d5ddf9ba8b57de --- M pywikibot/flow.py M pywikibot/site.py M tests/flow_tests.py 3 files changed, 429 insertions(+), 31 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/flow.py b/pywikibot/flow.py index 8645532..c9fe320 100644 --- a/pywikibot/flow.py +++ b/pywikibot/flow.py @@ -11,7 +11,16 @@ import logging +from pywikibot.exceptions import NoPage, UnknownExtension from pywikibot.page import BasePage +from pywikibot.tools import PY2 + +if not PY2: + unicode = str + basestring = (str,) + from urllib.parse import urlparse, parse_qs +else: + from urlparse import urlparse, parse_qs logger = logging.getLogger('pywiki.wiki.flow') @@ -43,7 +52,7 @@ super(FlowPage, self).__init__(source, title) if not self.site.has_extension('Flow'): - raise ValueError('site is not Flow-enabled') + raise UnknownExtension('site is not Flow-enabled') def _load_uuid(self): """Load and save the UUID of the page.""" @@ -60,6 +69,13 @@ self._load_uuid() return self._uuid + def get(self, force=False, get_redirect=False, sysop=False): + if get_redirect or force or sysop: + raise NotImplementedError + + # TODO: Return more useful data + return self._data + class Board(FlowPage): @@ -70,6 +86,57 @@ if not hasattr(self, '_data'): self._data = self.site.load_board(self) return self._data + + def _parse_url(self, links): + """Parse a URL retrieved from the API.""" + rule = links['fwd'] + parsed_url = urlparse(rule['url']) + params = parse_qs(parsed_url.query) + new_params = {} + for key, value in params.items(): + if key != 'title': + key = key.replace('topiclist_', '').replace('-', '_') + if key == 'offset_dir': + new_params['reverse'] = (value == 'rev') + else: + new_params[key] = value + return new_params + + def topics(self, format='wikitext', limit=100, sort_by='newest', + offset=None, offset_uuid='', reverse=False, + include_offset=False, toc_only=False): + """Load this board's topics. + + @param format: The content format to request the data in. + @type format: str (either 'wikitext', 'html', or 'fixed-html') + @param limit: The number of topics to fetch in each request. + @type limit: int + @param sort_by: Algorithm to sort topics by. + @type sort_by: str (either 'newest' or 'updated') + @param offset: The timestamp to start at (when sortby is 'updated'). + @type offset: Timestamp or equivalent str + @param offset_uuid: The UUID to start at (when sortby is 'newest'). + @type offset_uuid: str (in the form of a UUID) + @param reverse: Whether to reverse the topic ordering. + @type reverse: bool + @param include_offset: Whether to include the offset topic. + @type include_offset: bool + @param toc_only: Whether to only include information for the TOC. + @type toc_only: bool + @return: A generator of this board's topics. + @rtype: generator of Topic objects + """ + data = self.site.load_topiclist(self, format=format, limit=limit, + sortby=sort_by, toconly=toc_only, + offset=offset, offset_id=offset_uuid, + reverse=reverse, + include_offset=include_offset) + while data['roots']: + for root in data['roots']: + topic = Topic.from_topiclist_data(self, root, data) + yield topic + cont_args = self._parse_url(data['links']['pagination']) + data = self.site.load_topiclist(self, **cont_args) class Topic(FlowPage): @@ -82,8 +149,52 @@ self._data = self.site.load_topic(self) return self._data + @classmethod + def from_topiclist_data(cls, board, root_uuid, topiclist_data): + """Create a Topic object from API data. -# Flow non-page-like objects (currently just posts) + @param board: The topic's parent Flow board + @type board: Board + @param root_uuid: The UUID of the topic and its root post + @type root_uuid: unicode + @param topiclist_data: The data returned by view-topiclist + @type topiclist_data: dict + @return: A Topic object derived from the supplied data + @rtype: Topic + @raise TypeError: any passed parameters have wrong types + @raise ValueError: the passed topiclist_data is missing required data + """ + if not isinstance(board, Board): + raise TypeError('board must be a pywikibot.flow.Board object.') + if not isinstance(root_uuid, basestring): + raise TypeError('Topic/root UUID must be a string.') + + topic = cls(board.site, 'Topic:' + root_uuid) + topic._root = Post.fromJSON(topic, root_uuid, topiclist_data) + topic._uuid = root_uuid + return topic + + @property + def root(self): + """The root post of this topic.""" + if not hasattr(self, '_root'): + self._root = Post.fromJSON(self, self.uuid, self._data) + return self._root + + def replies(self, format='wikitext', force=False): + """A list of replies to this topic's root post. + + @param format: Content format to return contents in + @type format: str ('wikitext', 'html', or 'fixed-html') + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @return: The replies of this topic's root post + @rtype: list of Posts + """ + return self.root.replies(format=format, force=force) + + +# Flow non-page-like objects class Post(object): """A post to a Flow discussion topic.""" @@ -92,22 +203,81 @@ """ Constructor. - @param page: Flow topic or board - @type page: FlowPage + @param page: Flow topic + @type page: Topic @param uuid: UUID of a Flow post @type uuid: unicode @raise TypeError: incorrect types of parameters - @raise ValueError: use of non-Flow-enabled Site or invalid UUID """ - if not isinstance(page, FlowPage): - raise TypeError('page must be a FlowPage object') - - if not uuid: - raise ValueError('post UUID must be provided') + if not isinstance(page, Topic): + raise TypeError('Page must be a Topic object') + if not page.exists(): + raise NoPage(page, 'Topic must exist: %s') + if not isinstance(uuid, basestring): + raise TypeError('Post UUID must be a string') self._page = page self._uuid = uuid + + self._content = {} + + @classmethod + def fromJSON(cls, page, post_uuid, data): + """ + Create a Post object using the data returned from the API call. + + @param page: A Flow topic + @type page: Topic + @param post_uuid: The UUID of the post + @type post_uuid: unicode + @param data: The JSON data returned from the API + @type data: dict + + @return: A Post object + @raise TypeError: data is not a dict + @raise ValueError: data is missing required entries + """ + post = cls(page, post_uuid) + post._set_data(data) + + return post + + def _set_data(self, data): + """Set internal data and cache content. + + @param data: The data to store internally + @type data: dict + @raise TypeError: data is not a dict + @raise ValueError: missing data entries or post/revision not found + """ + if not isinstance(data, dict): + raise TypeError('Illegal post data (must be a dictionary).') + if ('posts' not in data) or ('revisions' not in data): + raise ValueError('Illegal post data (missing required data).') + if self.uuid not in data['posts']: + raise ValueError('Post not found in supplied data.') + + self._data = data + current_revision_id = data['posts'][self.uuid][0] + + if current_revision_id not in data['revisions']: + raise ValueError('Current revision of post' + 'not found in supplied data.') + + self._current_revision = data['revisions'][current_revision_id] + if 'content' in self._current_revision: + content = self._current_revision.pop('content') + assert isinstance(content, dict) + assert isinstance(content['content'], unicode) + self._content[content['format']] = content['content'] + + def _load(self, format='wikitext'): + """Load and cache the Post's data using the given content format.""" + data = self.site.load_post_current_revision(self.page, self.uuid, + format) + self._set_data(data) + return self._data @property def uuid(self): @@ -135,3 +305,47 @@ @rtype: FlowPage """ return self._page + + def get(self, format='wikitext', force=False, sysop=False): + """Return the contents of the post in the given format. + + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @param sysop: Whether to load using sysop rights. Implies force. + @type sysop: bool + @param format: Content format to return contents in + @type format: unicode + @return: The contents of the post in the given content format + @rtype: unicode + @raise NotImplementedError: use of 'sysop' + """ + if sysop: + raise NotImplementedError + + if format not in self._content or force: + self._load(format) + return self._content[format] + + def replies(self, format='wikitext', force=False): + """Return this post's replies. + + @param format: Content format to return contents in + @type format: str ('wikitext', 'html', or 'fixed-html') + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @return This post's replies + @rtype: list of Posts + """ + if format not in ('wikitext', 'html', 'fixed-html'): + raise ValueError('Invalid content format.') + + if hasattr(self, '_replies') and not force: + return self._replies + + if not hasattr(self, '_current_revision') or force: + self._load(format) + + reply_uuids = self._current_revision['replies'] + self._replies = [Post(self.page, uuid) for uuid in reply_uuids] + + return self._replies diff --git a/pywikibot/site.py b/pywikibot/site.py index 8208363..5ba136e 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -5771,11 +5771,52 @@ @param page: A Flow board @type page: Board - @return: A dict representing the board's data. + @return: A dict representing the board's metadata. @rtype: dict """ req = self._simple_request(action='flow', page=page, - submodule='view-topiclist') + submodule='view-topiclist', + vtllimit=1) + data = req.submit() + return data['flow']['view-topiclist']['result']['topiclist'] + + @need_extension('Flow') + def load_topiclist(self, page, format='wikitext', limit=100, + sortby='newest', toconly=False, offset=None, + offset_id=None, reverse=False, include_offset=False): + """Retrieve the topiclist of a Flow board. + + @param page: A Flow board + @type page: Board + @param format: The content format to request the data in. + @type format: str (either 'wikitext', 'html', or 'fixed-html') + @param limit: The number of topics to fetch in each request. + @type limit: int + @param sortby: Algorithm to sort topics by. + @type sortby: str (either 'newest' or 'updated') + @param toconly: Whether to only include information for the TOC. + @type toconly: bool + @param offset: The timestamp to start at (when sortby is 'updated'). + @type offset: Timestamp or equivalent str + @param offset_id: The topic UUID to start at (when sortby is 'newest'). + @type offset_id: str (in the form of a UUID) + @param reverse: Whether to reverse the topic ordering. + @type reverse: bool + @param include_offset: Whether to include the offset topic. + @type include_offset: bool + @return: A dict representing the board's topiclist. + @rtype: dict + """ + if offset: + offset = pywikibot.Timestamp.fromtimestampformat(offset) + offset_dir = reverse and 'rev' or 'fwd' + + params = {'action': 'flow', 'submodule': 'view-topiclist', 'page': page, + 'vtlformat': format, 'vtlsortby': sortby, + 'vtllimit': limit, 'vtloffset-dir': offset_dir, + 'vtloffset': offset, 'vtloffset-id': offset_id, + 'vtlinclude-offset': include_offset, 'vtltoconly': toconly} + req = self._request(parameters=params) data = req.submit() return data['flow']['view-topiclist']['result']['topiclist'] @@ -5793,6 +5834,25 @@ data = req.submit() return data['flow']['view-topic']['result']['topic'] + @need_extension('Flow') + def load_post_current_revision(self, page, post_id, format): + """Retrieve the data for a post to a Flow topic. + + @param page: A Flow topic + @type page: Topic + @param post_id: The UUID of the Post + @type post_id: unicode + @param format: The content format used for the returned content + @type format: unicode (either 'wikitext', 'html', or 'fixed-html') + @return: A dict representing the post data for the given UUID. + @rtype: dict + """ + req = self._simple_request(action='flow', page=page, + submodule='view-post', vppostId=post_id, + vpformat=format) + data = req.submit() + return data['flow']['view-post']['result']['topic'] + def watched_pages(self, sysop=False, force=False, step=None, total=None): """ Return watchlist. diff --git a/tests/flow_tests.py b/tests/flow_tests.py index 8c4f85d..5ba6b7b 100644 --- a/tests/flow_tests.py +++ b/tests/flow_tests.py @@ -9,8 +9,9 @@ __version__ = '$Id$' -import pywikibot -import pywikibot.flow +from pywikibot.exceptions import NoPage +from pywikibot.flow import Board, Topic, Post +from pywikibot.tools import PY2 from tests.aspects import ( TestCase, @@ -21,29 +22,55 @@ BasePageLoadRevisionsCachingTestBase, ) +if not PY2: + unicode = str + class TestBoardBasePageMethods(BasePageMethodsTestBase): - """Test Flow pages using BasePage-defined methods.""" + """Test Flow board pages using BasePage-defined methods.""" family = 'mediawiki' code = 'mediawiki' def setUp(self): """Set up unit test.""" - self._page = pywikibot.flow.Board( - self.site, 'Talk:Sandbox') + self._page = Board(self.site, 'Talk:Sandbox') super(TestBoardBasePageMethods, self).setUp() def test_basepage_methods(self): - """Test basic Page methods on a Flow page.""" + """Test basic Page methods on a Flow board page.""" self._test_invoke() self._test_return_datatypes() - self.assertEqual(self._page.isRedirectPage(), False) + self.assertFalse(self._page.isRedirectPage()) self.assertEqual(self._page.latest_revision.parent_id, 0) def test_content_model(self): """Test Flow page content model.""" + self.assertEqual(self._page.content_model, 'flow-board') + + +class TestTopicBasePageMethods(BasePageMethodsTestBase): + + """Test Flow topic pages using BasePage-defined methods.""" + + family = 'mediawiki' + code = 'mediawiki' + + def setUp(self): + """Set up unit test.""" + self._page = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + super(TestTopicBasePageMethods, self).setUp() + + def test_basepage_methods(self): + """Test basic Page methods on a Flow topic page.""" + self._test_invoke() + self._test_return_datatypes() + self.assertFalse(self._page.isRedirectPage()) + self.assertEqual(self._page.latest_revision.parent_id, 0) + + def test_content_model(self): + """Test Flow topic page content model.""" self.assertEqual(self._page.content_model, 'flow-board') @@ -56,12 +83,12 @@ def setUp(self): """Set up unit test.""" - self._page = pywikibot.flow.Board( - self.site, 'Talk:Sandbox') + self._page = Board(self.site, 'Talk:Sandbox') super(TestLoadRevisionsCaching, self).setUp() def test_page_text(self): """Test site.loadrevisions() with Page.text.""" + self.skipTest('See T107537') self._test_page_text() @@ -76,15 +103,13 @@ def test_board_uuid(self): """Test retrieval of Flow board UUID.""" - site = self.get_site() - board = pywikibot.flow.Board(site, u'Talk:Sandbox') - self.assertEqual(board.uuid, u'rl7iby6wgksbpfno') + board = Board(self.site, 'Talk:Sandbox') + self.assertEqual(board.uuid, 'rl7iby6wgksbpfno') def test_topic_uuid(self): """Test retrieval of Flow topic UUID.""" - site = self.get_site() - topic = pywikibot.flow.Topic(site, u'Topic:Sh6wgo5tu3qui1w2') - self.assertEqual(topic.uuid, u'sh6wgo5tu3qui1w2') + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + self.assertEqual(topic.uuid, 'sh6wgo5tu3qui1w2') def test_post_uuid(self): """Test retrieval of Flow post UUID. @@ -93,7 +118,106 @@ the property to make sure the UUID passed to the constructor is stored properly. """ - site = self.get_site() - topic = pywikibot.flow.Topic(site, u'Topic:Sh6wgo5tu3qui1w2') - post = pywikibot.flow.Post(topic, u'sh6wgoagna97q0ia') - self.assertEqual(post.uuid, u'sh6wgoagna97q0ia') + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + post = Post(topic, 'sh6wgoagna97q0ia') + self.assertEqual(post.uuid, 'sh6wgoagna97q0ia') + + def test_post_contents(self): + """Test retrieval of Flow post contents.""" + # Load + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + post = Post(topic, 'sh6wgoagna97q0ia') + # Wikitext + wikitext = post.get(format='wikitext') + self.assertIn('wikitext', post._content) + self.assertNotIn('html', post._content) + self.assertIsInstance(wikitext, unicode) + self.assertNotEqual(wikitext, '') + # HTML + html = post.get(format='html') + self.assertIn('html', post._content) + self.assertIn('wikitext', post._content) + self.assertIsInstance(html, unicode) + self.assertNotEqual(html, '') + # Caching (hit) + post._content['html'] = 'something' + html = post.get(format='html') + self.assertIsInstance(html, unicode) + self.assertEqual(html, 'something') + self.assertIn('html', post._content) + # Caching (reload) + post._content['html'] = 'something' + html = post.get(format='html', force=True) + self.assertIsInstance(html, unicode) + self.assertNotEqual(html, 'something') + self.assertIn('html', post._content) + + def test_topiclist(self): + """Test loading of topiclist.""" + board = Board(self.site, 'Talk:Sandbox') + i = 0 + for topic in board.topics(limit=7): + i += 1 + if i == 10: + break + self.assertEqual(i, 10) + + +class TestFlowFactoryErrors(TestCase): + + """Test errors associated with class methods generating Flow objects.""" + + family = 'test' + code = 'test' + + cached = True + + def test_illegal_arguments(self): + """Test illegal method arguments.""" + board = Board(self.site, 'Talk:Pywikibot test') + real_topic = Topic(self.site, 'Topic:Slbktgav46omarsd') + fake_topic = Topic(self.site, 'Topic:Abcdefgh12345678') + # Topic.from_topiclist_data + self.assertRaises(TypeError, Topic.from_topiclist_data, self.site, '', {}) + self.assertRaises(TypeError, Topic.from_topiclist_data, board, 521, {}) + self.assertRaises(TypeError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', [0, 1, 2]) + self.assertRaises(NoPage, Topic.from_topiclist_data, board, + 'abc', {'stuff': 'blah'}) + + # Post.fromJSON + self.assertRaises(TypeError, Post.fromJSON, board, 'abc', {}) + self.assertRaises(TypeError, Post.fromJSON, real_topic, 1234, {}) + self.assertRaises(TypeError, Post.fromJSON, real_topic, 'abc', []) + self.assertRaises(NoPage, Post.fromJSON, fake_topic, 'abc', + {'posts': [], 'revisions': []}) + + def test_invalid_data(self): + """Test invalid "API" data.""" + board = Board(self.site, 'Talk:Pywikibot test') + real_topic = Topic(self.site, 'Topic:Slbktgav46omarsd') + # Topic.from_topiclist_data + self.assertRaises(ValueError, Topic.from_topiclist_data, + board, 'slbktgav46omarsd', {'stuff': 'blah'}) + self.assertRaises(ValueError, Topic.from_topiclist_data, + board, 'slbktgav46omarsd', + {'posts': [], 'revisions': []}) + self.assertRaises(ValueError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', + {'posts': {'slbktgav46omarsd': ['123']}, + 'revisions': {'456': []}}) + self.assertRaises(AssertionError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', + {'posts': {'slbktgav46omarsd': ['123']}, + 'revisions': {'123': {'content': 789}}}) + + # Post.fromJSON + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', {}) + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', + {'stuff': 'blah'}) + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', + {'posts': {'abc': ['123']}, + 'revisions': {'456': []}}) + self.assertRaises(AssertionError, Post.fromJSON, real_topic, 'abc', + {'posts': {'abc': ['123']}, + 'revisions': {'123': {'content': 789}}}) -- To view, visit https://gerrit.wikimedia.org/r/217717 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ibd84d0763a7745da6410c194b8d5ddf9ba8b57de Gerrit-PatchSet: 38 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Happy5214 <happy5214(a)gmail.com> Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Mattflaschen <mflaschen(a)wikimedia.org> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 10 months

← Newer
1
...
17
18
19
20
21
22
23
24
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits August 2015