[Pywikibot-commits] [Gerrit] ...core[master]: Move Siteinfo to its own _siteinfo.py file

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/616320 ) Change subject: Move Siteinfo to its own _siteinfo.py file ...................................................................... Move Siteinfo to its own _siteinfo.py file Also move siteninfo tests from site_tests.py to new siteinfo_tests.py No source changes where made. site/__init__.py is the biggest framework file with 335 KB disk space and 8280 lines of code. Currently it contains 13 classes. This should be splitted a bit more into smaller parts for readability and maintainability. Change-Id: I8a87dc2bf0b178aa862448153dbffa3fcfe81f79 --- M pywikibot/CONTENT.rst M pywikibot/site/__init__.py A pywikibot/site/_siteinfo.py M tests/__init__.py M tests/site_tests.py A tests/siteinfo_tests.py 6 files changed, 492 insertions(+), 445 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/CONTENT.rst b/pywikibot/CONTENT.rst index 30f92ee..de541b7 100644 --- a/pywikibot/CONTENT.rst +++ b/pywikibot/CONTENT.rst @@ -113,6 +113,8 @@ +----------------------------+------------------------------------------------------+ | _decorators.py | Decorators used by site models. | +----------------------------+------------------------------------------------------+ + | _siteinfo.py | Objects representing site info data contents. | + +----------------------------+------------------------------------------------------+ +----------------------------+------------------------------------------------------+ diff --git a/pywikibot/site/__init__.py b/pywikibot/site/__init__.py index edfbf11..d090257 100644 --- a/pywikibot/site/__init__.py +++ b/pywikibot/site/__init__.py @@ -10,7 +10,6 @@ # # Distributed under the terms of the MIT license. # -import copy import datetime import functools import heapq @@ -25,12 +24,11 @@ import uuid from collections import defaultdict, namedtuple -from collections.abc import Iterable, Container, Mapping +from collections.abc import Iterable, Mapping from contextlib import suppress from enum import IntEnum from itertools import zip_longest from textwrap import fill -from typing import Optional from warnings import warn import pywikibot @@ -68,6 +66,7 @@ UnknownSite, ) from pywikibot.site._decorators import need_extension, need_right, need_version +from pywikibot.site._siteinfo import Siteinfo from pywikibot.throttle import Throttle from pywikibot.tools import ( ComparableMixin, @@ -1208,345 +1207,6 @@ return api.encode_url(query) -class Siteinfo(Container): - - """ - A 'dictionary' like container for siteinfo. - - This class queries the server to get the requested siteinfo property. - Optionally it can cache this directly in the instance so that later - requests don't need to query the server. - - All values of the siteinfo property 'general' are directly available. - """ - - WARNING_REGEX = re.compile(r'Unrecognized values? for parameter ' - r'["\']siprop["\']: (.+?)\.?$') - - # Until we get formatversion=2, we have to convert empty-string properties - # into booleans so they are easier to use. - BOOLEAN_PROPS = { - 'general': [ - 'imagewhitelistenabled', - 'langconversion', - 'titleconversion', - 'rtl', - 'readonly', - 'writeapi', - 'variantarticlepath', - 'misermode', - 'uploadsenabled', - ], - 'namespaces': [ # for each namespace - 'subpages', - 'content', - 'nonincludable', - ], - 'magicwords': [ # for each magicword - 'case-sensitive', - ], - } - - def __init__(self, site): - """Initialise it with an empty cache.""" - self._site = site - self._cache = {} - - @staticmethod - def _get_default(key: str): - """ - Return the default value for different properties. - - If the property is 'restrictions' it returns a dictionary with: - - 'cascadinglevels': 'sysop' - - 'semiprotectedlevels': 'autoconfirmed' - - 'levels': '' (everybody), 'autoconfirmed', 'sysop' - - 'types': 'create', 'edit', 'move', 'upload' - Otherwise it returns L{pywikibot.tools.EMPTY_DEFAULT}. - - @param key: The property name - @return: The default value - @rtype: dict or L{pywikibot.tools.EmptyDefault} - """ - if key == 'restrictions': - # implemented in b73b5883d486db0e9278ef16733551f28d9e096d - return { - 'cascadinglevels': ['sysop'], - 'semiprotectedlevels': ['autoconfirmed'], - 'levels': ['', 'autoconfirmed', 'sysop'], - 'types': ['create', 'edit', 'move', 'upload'] - } - - if key == 'fileextensions': - # the default file extensions in MediaWiki - return [{'ext': ext} for ext in ['png', 'gif', 'jpg', 'jpeg']] - - return pywikibot.tools.EMPTY_DEFAULT - - @staticmethod - def _post_process(prop, data): - """Do some default handling of data. Directly modifies data.""" - # Be careful with version tests inside this here as it might need to - # query this method to actually get the version number - - # Convert boolean props from empty strings to actual boolean values - if prop in Siteinfo.BOOLEAN_PROPS.keys(): - # siprop=namespaces and - # magicwords has properties per item in result - if prop in ('namespaces', 'magicwords'): - for index, value in enumerate(data): - # namespaces uses a dict, while magicwords uses a list - key = index if type(data) is list else value - for p in Siteinfo.BOOLEAN_PROPS[prop]: - data[key][p] = p in data[key] - else: - for p in Siteinfo.BOOLEAN_PROPS[prop]: - data[p] = p in data - - def _get_siteinfo(self, prop, expiry) -> dict: - """ - Retrieve a siteinfo property. - - All properties which the site doesn't - support contain the default value. Because pre-1.12 no data was - returned when a property doesn't exists, it queries each property - independetly if a property is invalid. - - @param prop: The property names of the siteinfo. - @type prop: str or iterable - @param expiry: The expiry date of the cached request. - @type expiry: int (days), L{datetime.timedelta}, False (config) - @return: A dictionary with the properties of the site. Each entry in - the dictionary is a tuple of the value and a boolean to save if it - is the default value. - @see: U{https://www.mediawiki.org/wiki/API:Meta#siteinfo_.2F_si} - """ - def warn_handler(mod, message): - """Return True if the warning is handled.""" - matched = Siteinfo.WARNING_REGEX.match(message) - if mod == 'siteinfo' and matched: - invalid_properties.extend( - prop.strip() for prop in matched.group(1).split(',')) - return True - else: - return False - - props = [prop] if isinstance(prop, str) else prop - if not props: - raise ValueError('At least one property name must be provided.') - - invalid_properties = [] - request = self._site._request( - expiry=pywikibot.config.API_config_expiry - if expiry is False else expiry, - parameters={ - 'action': 'query', 'meta': 'siteinfo', 'siprop': props, - } - ) - # With 1.25wmf5 it'll require continue or rawcontinue. As we don't - # continue anyway we just always use continue. - request['continue'] = True - # warnings are handled later - request._warning_handler = warn_handler - try: - data = request.submit() - except api.APIError as e: - if e.code == 'siunknown_siprop': - if len(props) == 1: - pywikibot.log( - "Unable to get siprop '{0}'".format(props[0])) - return {props[0]: (Siteinfo._get_default(props[0]), False)} - else: - pywikibot.log('Unable to get siteinfo, because at least ' - "one property is unknown: '{0}'".format( - "', '".join(props))) - results = {} - for prop in props: - results.update(self._get_siteinfo(prop, expiry)) - return results - raise - else: - result = {} - if invalid_properties: - for prop in invalid_properties: - result[prop] = (Siteinfo._get_default(prop), False) - pywikibot.log("Unable to get siprop(s) '{0}'".format( - "', '".join(invalid_properties))) - if 'query' in data: - # If the request is a CachedRequest, use the _cachetime attr. - cache_time = getattr( - request, '_cachetime', None) or datetime.datetime.utcnow() - for prop in props: - if prop in data['query']: - self._post_process(prop, data['query'][prop]) - result[prop] = (data['query'][prop], cache_time) - return result - - @staticmethod - def _is_expired(cache_date, expire): - """Return true if the cache date is expired.""" - if isinstance(expire, bool): - return expire - - if not cache_date: # default values are always expired - return True - - # cached date + expiry are in the past if it's expired - return cache_date + expire < datetime.datetime.utcnow() - - def _get_general(self, key: str, expiry): - """ - Return a siteinfo property which is loaded by default. - - The property 'general' will be queried if it wasn't yet or it's forced. - Additionally all uncached default properties are queried. This way - multiple default properties are queried with one request. It'll cache - always all results. - - @param key: The key to search for. - @param expiry: If the cache is older than the expiry it ignores the - cache and queries the server to get the newest value. - @type expiry: int (days), L{datetime.timedelta}, False (never) - @return: If that property was retrieved via this method. Returns None - if the key was not in the retrieved values. - @rtype: various (the value), bool (if the default value is used) - """ - if 'general' not in self._cache: - pywikibot.debug('general siteinfo not loaded yet.', _logger) - force = True - props = ['namespaces', 'namespacealiases'] - else: - force = Siteinfo._is_expired(self._cache['general'][1], expiry) - props = [] - if force: - props = [prop for prop in props if prop not in self._cache] - if props: - pywikibot.debug( - "Load siteinfo properties '{0}' along with 'general'" - .format("', '".join(props)), _logger) - props += ['general'] - default_info = self._get_siteinfo(props, expiry) - for prop in props: - self._cache[prop] = default_info[prop] - if key in default_info: - return default_info[key] - if key in self._cache['general'][0]: - return self._cache['general'][0][key], self._cache['general'] - else: - return None - - def __getitem__(self, key: str): - """Return a siteinfo property, caching and not forcing it.""" - return self.get(key, False) # caches and doesn't force it - - def get(self, key: str, get_default: bool = True, cache: bool = True, - expiry=False): - """ - Return a siteinfo property. - - It will never throw an APIError if it only stated, that the siteinfo - property doesn't exist. Instead it will use the default value. - - @param key: The name of the siteinfo property. - @param get_default: Whether to throw an KeyError if the key is invalid. - @param cache: Caches the result internally so that future accesses via - this method won't query the server. - @param expiry: If the cache is older than the expiry it ignores the - cache and queries the server to get the newest value. - @type expiry: int/float (days), L{datetime.timedelta}, - False (never expired), True (always expired) - @return: The gathered property - @rtype: various - @raises KeyError: If the key is not a valid siteinfo property and the - get_default option is set to False. - @see: L{_get_siteinfo} - """ - # If expiry is a float or int convert to timedelta - # Note: bool is an instance of int - if isinstance(expiry, float) or type(expiry) == int: - expiry = datetime.timedelta(expiry) - - # expire = 0 (or timedelta(0)) are always expired and their bool is - # False, so skip them EXCEPT if it's literally False, then they expire - # never. - if expiry and expiry is not True or expiry is False: - try: - cached = self._get_cached(key) - except KeyError: - pass - else: # cached value available - # is a default value, but isn't accepted - if not cached[1] and not get_default: - raise KeyError(key) - if not Siteinfo._is_expired(cached[1], expiry): - return copy.deepcopy(cached[0]) - - preloaded = self._get_general(key, expiry) - if not preloaded: - preloaded = self._get_siteinfo(key, expiry)[key] - else: - cache = False - - if not preloaded[1] and not get_default: - raise KeyError(key) - - if cache: - self._cache[key] = preloaded - - return copy.deepcopy(preloaded[0]) - - def _get_cached(self, key: str): - """Return the cached value or a KeyError exception if not cached.""" - if 'general' in self._cache: - if key in self._cache['general'][0]: - return (self._cache['general'][0][key], - self._cache['general'][1]) - else: - return self._cache[key] - raise KeyError(key) - - def __contains__(self, key: str) -> bool: - """Return whether the value is cached.""" - try: - self._get_cached(key) - except KeyError: - return False - else: - return True - - def is_recognised(self, key: str) -> Optional[bool]: - """Return if 'key' is a valid property name. 'None' if not cached.""" - time = self.get_requested_time(key) - return None if time is None else bool(time) - - def get_requested_time(self, key: str): - """ - Return when 'key' was successfully requested from the server. - - If the property is actually in the siprop 'general' it returns the - last request from the 'general' siprop. - - @param key: The siprop value or a property of 'general'. - @return: The last time the siprop of 'key' was requested. - @rtype: None (never), False (default), L{datetime.datetime} (cached) - """ - with suppress(KeyError): - return self._get_cached(key)[1] - - return None - - def __call__(self, key='general', force=False, dump=False): - """DEPRECATED: Return the entry for key or dump the complete cache.""" - issue_deprecation_warning( - 'Calling siteinfo', 'itself as a dictionary', since='20161221' - ) - result = self.get(key, expiry=force) - if not dump: - return result - else: - return self._cache - - class TokenWallet: """Container for tokens.""" diff --git a/pywikibot/site/_siteinfo.py b/pywikibot/site/_siteinfo.py new file mode 100644 index 0000000..28f13cc --- /dev/null +++ b/pywikibot/site/_siteinfo.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- +"""Objects representing site info data contents.""" +# +# (C) Pywikibot team, 2008-2020 +# +# Distributed under the terms of the MIT license. +# +import copy +import datetime +import re + +from collections.abc import Container +from contextlib import suppress +from typing import Optional + +import pywikibot + +from pywikibot.data import api +from pywikibot.tools import EMPTY_DEFAULT, issue_deprecation_warning + + +_logger = 'wiki.siteinfo' + + +class Siteinfo(Container): + + """ + A 'dictionary' like container for siteinfo. + + This class queries the server to get the requested siteinfo property. + Optionally it can cache this directly in the instance so that later + requests don't need to query the server. + + All values of the siteinfo property 'general' are directly available. + """ + + WARNING_REGEX = re.compile(r'Unrecognized values? for parameter ' + r'["\']siprop["\']: (.+?)\.?$') + + # Until we get formatversion=2, we have to convert empty-string properties + # into booleans so they are easier to use. + BOOLEAN_PROPS = { + 'general': [ + 'imagewhitelistenabled', + 'langconversion', + 'titleconversion', + 'rtl', + 'readonly', + 'writeapi', + 'variantarticlepath', + 'misermode', + 'uploadsenabled', + ], + 'namespaces': [ # for each namespace + 'subpages', + 'content', + 'nonincludable', + ], + 'magicwords': [ # for each magicword + 'case-sensitive', + ], + } + + def __init__(self, site): + """Initialise it with an empty cache.""" + self._site = site + self._cache = {} + + @staticmethod + def _get_default(key: str): + """ + Return the default value for different properties. + + If the property is 'restrictions' it returns a dictionary with: + - 'cascadinglevels': 'sysop' + - 'semiprotectedlevels': 'autoconfirmed' + - 'levels': '' (everybody), 'autoconfirmed', 'sysop' + - 'types': 'create', 'edit', 'move', 'upload' + Otherwise it returns L{pywikibot.tools.EMPTY_DEFAULT}. + + @param key: The property name + @return: The default value + @rtype: dict or L{pywikibot.tools.EmptyDefault} + """ + if key == 'restrictions': + # implemented in b73b5883d486db0e9278ef16733551f28d9e096d + return { + 'cascadinglevels': ['sysop'], + 'semiprotectedlevels': ['autoconfirmed'], + 'levels': ['', 'autoconfirmed', 'sysop'], + 'types': ['create', 'edit', 'move', 'upload'] + } + + if key == 'fileextensions': + # the default file extensions in MediaWiki + return [{'ext': ext} for ext in ['png', 'gif', 'jpg', 'jpeg']] + + return EMPTY_DEFAULT + + @staticmethod + def _post_process(prop, data): + """Do some default handling of data. Directly modifies data.""" + # Be careful with version tests inside this here as it might need to + # query this method to actually get the version number + + # Convert boolean props from empty strings to actual boolean values + if prop in Siteinfo.BOOLEAN_PROPS.keys(): + # siprop=namespaces and + # magicwords has properties per item in result + if prop in ('namespaces', 'magicwords'): + for index, value in enumerate(data): + # namespaces uses a dict, while magicwords uses a list + key = index if type(data) is list else value + for p in Siteinfo.BOOLEAN_PROPS[prop]: + data[key][p] = p in data[key] + else: + for p in Siteinfo.BOOLEAN_PROPS[prop]: + data[p] = p in data + + def _get_siteinfo(self, prop, expiry) -> dict: + """ + Retrieve a siteinfo property. + + All properties which the site doesn't + support contain the default value. Because pre-1.12 no data was + returned when a property doesn't exists, it queries each property + independetly if a property is invalid. + + @param prop: The property names of the siteinfo. + @type prop: str or iterable + @param expiry: The expiry date of the cached request. + @type expiry: int (days), L{datetime.timedelta}, False (config) + @return: A dictionary with the properties of the site. Each entry in + the dictionary is a tuple of the value and a boolean to save if it + is the default value. + @see: U{https://www.mediawiki.org/wiki/API:Meta#siteinfo_.2F_si} + """ + def warn_handler(mod, message): + """Return True if the warning is handled.""" + matched = Siteinfo.WARNING_REGEX.match(message) + if mod == 'siteinfo' and matched: + invalid_properties.extend( + prop.strip() for prop in matched.group(1).split(',')) + return True + else: + return False + + props = [prop] if isinstance(prop, str) else prop + if not props: + raise ValueError('At least one property name must be provided.') + + invalid_properties = [] + request = self._site._request( + expiry=pywikibot.config.API_config_expiry + if expiry is False else expiry, + parameters={ + 'action': 'query', 'meta': 'siteinfo', 'siprop': props, + } + ) + # With 1.25wmf5 it'll require continue or rawcontinue. As we don't + # continue anyway we just always use continue. + request['continue'] = True + # warnings are handled later + request._warning_handler = warn_handler + try: + data = request.submit() + except api.APIError as e: + if e.code == 'siunknown_siprop': + if len(props) == 1: + pywikibot.log( + "Unable to get siprop '{0}'".format(props[0])) + return {props[0]: (Siteinfo._get_default(props[0]), False)} + else: + pywikibot.log('Unable to get siteinfo, because at least ' + "one property is unknown: '{0}'".format( + "', '".join(props))) + results = {} + for prop in props: + results.update(self._get_siteinfo(prop, expiry)) + return results + raise + else: + result = {} + if invalid_properties: + for prop in invalid_properties: + result[prop] = (Siteinfo._get_default(prop), False) + pywikibot.log("Unable to get siprop(s) '{0}'".format( + "', '".join(invalid_properties))) + if 'query' in data: + # If the request is a CachedRequest, use the _cachetime attr. + cache_time = getattr( + request, '_cachetime', None) or datetime.datetime.utcnow() + for prop in props: + if prop in data['query']: + self._post_process(prop, data['query'][prop]) + result[prop] = (data['query'][prop], cache_time) + return result + + @staticmethod + def _is_expired(cache_date, expire): + """Return true if the cache date is expired.""" + if isinstance(expire, bool): + return expire + + if not cache_date: # default values are always expired + return True + + # cached date + expiry are in the past if it's expired + return cache_date + expire < datetime.datetime.utcnow() + + def _get_general(self, key: str, expiry): + """ + Return a siteinfo property which is loaded by default. + + The property 'general' will be queried if it wasn't yet or it's forced. + Additionally all uncached default properties are queried. This way + multiple default properties are queried with one request. It'll cache + always all results. + + @param key: The key to search for. + @param expiry: If the cache is older than the expiry it ignores the + cache and queries the server to get the newest value. + @type expiry: int (days), L{datetime.timedelta}, False (never) + @return: If that property was retrieved via this method. Returns None + if the key was not in the retrieved values. + @rtype: various (the value), bool (if the default value is used) + """ + if 'general' not in self._cache: + pywikibot.debug('general siteinfo not loaded yet.', _logger) + force = True + props = ['namespaces', 'namespacealiases'] + else: + force = Siteinfo._is_expired(self._cache['general'][1], expiry) + props = [] + if force: + props = [prop for prop in props if prop not in self._cache] + if props: + pywikibot.debug( + "Load siteinfo properties '{0}' along with 'general'" + .format("', '".join(props)), _logger) + props += ['general'] + default_info = self._get_siteinfo(props, expiry) + for prop in props: + self._cache[prop] = default_info[prop] + if key in default_info: + return default_info[key] + if key in self._cache['general'][0]: + return self._cache['general'][0][key], self._cache['general'] + else: + return None + + def __getitem__(self, key: str): + """Return a siteinfo property, caching and not forcing it.""" + return self.get(key, False) # caches and doesn't force it + + def get(self, key: str, get_default: bool = True, cache: bool = True, + expiry=False): + """ + Return a siteinfo property. + + It will never throw an APIError if it only stated, that the siteinfo + property doesn't exist. Instead it will use the default value. + + @param key: The name of the siteinfo property. + @param get_default: Whether to throw an KeyError if the key is invalid. + @param cache: Caches the result internally so that future accesses via + this method won't query the server. + @param expiry: If the cache is older than the expiry it ignores the + cache and queries the server to get the newest value. + @type expiry: int/float (days), L{datetime.timedelta}, + False (never expired), True (always expired) + @return: The gathered property + @rtype: various + @raises KeyError: If the key is not a valid siteinfo property and the + get_default option is set to False. + @see: L{_get_siteinfo} + """ + # If expiry is a float or int convert to timedelta + # Note: bool is an instance of int + if isinstance(expiry, float) or type(expiry) == int: + expiry = datetime.timedelta(expiry) + + # expire = 0 (or timedelta(0)) are always expired and their bool is + # False, so skip them EXCEPT if it's literally False, then they expire + # never. + if expiry and expiry is not True or expiry is False: + try: + cached = self._get_cached(key) + except KeyError: + pass + else: # cached value available + # is a default value, but isn't accepted + if not cached[1] and not get_default: + raise KeyError(key) + if not Siteinfo._is_expired(cached[1], expiry): + return copy.deepcopy(cached[0]) + + preloaded = self._get_general(key, expiry) + if not preloaded: + preloaded = self._get_siteinfo(key, expiry)[key] + else: + cache = False + + if not preloaded[1] and not get_default: + raise KeyError(key) + + if cache: + self._cache[key] = preloaded + + return copy.deepcopy(preloaded[0]) + + def _get_cached(self, key: str): + """Return the cached value or a KeyError exception if not cached.""" + if 'general' in self._cache: + if key in self._cache['general'][0]: + return (self._cache['general'][0][key], + self._cache['general'][1]) + else: + return self._cache[key] + raise KeyError(key) + + def __contains__(self, key: str) -> bool: + """Return whether the value is cached.""" + try: + self._get_cached(key) + except KeyError: + return False + else: + return True + + def is_recognised(self, key: str) -> Optional[bool]: + """Return if 'key' is a valid property name. 'None' if not cached.""" + time = self.get_requested_time(key) + return None if time is None else bool(time) + + def get_requested_time(self, key: str): + """ + Return when 'key' was successfully requested from the server. + + If the property is actually in the siprop 'general' it returns the + last request from the 'general' siprop. + + @param key: The siprop value or a property of 'general'. + @return: The last time the siprop of 'key' was requested. + @rtype: None (never), False (default), L{datetime.datetime} (cached) + """ + with suppress(KeyError): + return self._get_cached(key)[1] + + return None + + def __call__(self, key='general', force=False, dump=False): + """DEPRECATED: Return the entry for key or dump the complete cache.""" + issue_deprecation_warning( + 'Calling siteinfo', 'itself as a dictionary', since='20161221' + ) + result = self.get(key, expiry=force) + if not dump: + return result + else: + return self._cache diff --git a/tests/__init__.py b/tests/__init__.py index fe69b79..b762659 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -105,6 +105,7 @@ 'site', 'site_decorators', 'site_detect', + 'siteinfo', 'sparql', 'tests', 'textlib', diff --git a/tests/site_tests.py b/tests/site_tests.py index 6286be3..20b94b7 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -7,23 +7,21 @@ # import pickle import random -import re import threading import time from collections.abc import Iterable, Mapping from contextlib import suppress -from datetime import datetime import pywikibot -from pywikibot import async_request, config, page_put_queue from pywikibot.comms import http +from pywikibot import config from pywikibot.data import api from pywikibot.exceptions import HiddenKeyError from pywikibot.tools import suppress_warnings -from tests import patch, unittest_print, MagicMock +from tests import patch, unittest_print from tests.aspects import ( AlteredDefaultSiteTestCase, DefaultDrySiteTestCase, @@ -37,7 +35,6 @@ WikidataTestCase, ) from tests.basepage import BasePageLoadRevisionsCachingTestBase -from tests.utils import entered_loop class TokenTestBase(TestCaseBase): @@ -2381,104 +2378,6 @@ self.assertLength(mypage._revisions, 12) -class TestSiteInfo(DefaultSiteTestCase): - - """Test cases for Site metadata and capabilities.""" - - cached = True - - def test_siteinfo(self): - """Test the siteinfo property.""" - # general enteries - mysite = self.get_site() - self.assertIsInstance(mysite.siteinfo['timeoffset'], (int, float)) - self.assertTrue(-12 * 60 <= mysite.siteinfo['timeoffset'] <= +14 * 60) - self.assertEqual(mysite.siteinfo['timeoffset'] % 15, 0) - self.assertRegex(mysite.siteinfo['timezone'], - '([A-Z]{3,4}|[A-Z][a-z]+/[A-Z][a-z]+)') - self.assertIn(mysite.siteinfo['case'], ['first-letter', - 'case-sensitive']) - self.assertIsInstance( - datetime.strptime(mysite.siteinfo['time'], '%Y-%m-%dT%H:%M:%SZ'), - datetime) - self.assertEqual(re.findall(r'\$1', mysite.siteinfo['articlepath']), - ['$1']) - - def test_siteinfo_boolean(self): - """Test conversion of boolean properties from empty strings.""" - mysite = self.get_site() - self.assertIsInstance(mysite.siteinfo['titleconversion'], bool) - - self.assertIsInstance(mysite.namespaces[0].subpages, bool) - self.assertIsInstance(mysite.namespaces[0].content, bool) - - def test_properties_with_defaults(self): - """Test the siteinfo properties with defaults.""" - # This does not test that the defaults work correct, - # unless the default site is a version needing these defaults - # 'fileextensions' introduced in v1.15: - self.assertIsInstance(self.site.siteinfo.get('fileextensions'), list) - self.assertIn('fileextensions', self.site.siteinfo) - fileextensions = self.site.siteinfo.get('fileextensions') - self.assertIn({'ext': 'png'}, fileextensions) - # 'restrictions' introduced in v1.23: - mysite = self.site - self.assertIsInstance(mysite.siteinfo.get('restrictions'), dict) - self.assertIn('restrictions', mysite.siteinfo) - restrictions = self.site.siteinfo.get('restrictions') - self.assertIn('cascadinglevels', restrictions) - - def test_no_cache(self): - """Test siteinfo caching can be disabled.""" - if 'fileextensions' in self.site.siteinfo._cache: - del self.site.siteinfo._cache['fileextensions'] - self.site.siteinfo.get('fileextensions', cache=False) - self.assertNotIn('fileextensions', self.site.siteinfo) - - def test_not_exists(self): - """Test accessing a property not in siteinfo.""" - not_exists = 'this-property-does-not-exist' - mysite = self.site - self.assertRaises(KeyError, mysite.siteinfo.__getitem__, not_exists) - self.assertNotIn(not_exists, mysite.siteinfo) - self.assertIsEmpty(mysite.siteinfo.get(not_exists)) - self.assertFalse(entered_loop(mysite.siteinfo.get(not_exists))) - self.assertFalse( - entered_loop(mysite.siteinfo.get(not_exists).items())) - self.assertFalse( - entered_loop(mysite.siteinfo.get(not_exists).values())) - self.assertFalse(entered_loop(mysite.siteinfo.get(not_exists).keys())) - - -class TestSiteinfoDry(DefaultDrySiteTestCase): - - """Test Siteinfo in dry mode.""" - - def test_siteinfo_timestamps(self): - """Test that cache has the timestamp of CachedRequest.""" - site = self.get_site() - request_mock = MagicMock() - request_mock.submit = lambda: {'query': {'_prop': '_value'}} - request_mock._cachetime = '_cache_time' - with patch.object(site, '_request', return_value=request_mock): - siteinfo = pywikibot.site.Siteinfo(site) - result = siteinfo._get_siteinfo('_prop', False) - self.assertEqual(result, {'_prop': ('_value', '_cache_time')}) - - -class TestSiteinfoAsync(DefaultSiteTestCase): - - """Test asynchronous siteinfo fetch.""" - - def test_async_request(self): - """Test async request.""" - self.assertTrue(page_put_queue.empty()) - self.assertNotIn('statistics', self.site.siteinfo) - async_request(self.site.siteinfo.get, 'statistics') - page_put_queue.join() - self.assertIn('statistics', self.site.siteinfo) - - class TestSiteLoadRevisionsCaching(BasePageLoadRevisionsCachingTestBase, DefaultSiteTestCase): diff --git a/tests/siteinfo_tests.py b/tests/siteinfo_tests.py new file mode 100644 index 0000000..6716f0a --- /dev/null +++ b/tests/siteinfo_tests.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +"""Tests for the site module.""" +# +# (C) Pywikibot team, 2008-2020 +# +# Distributed under the terms of the MIT license. +# +import re + +from contextlib import suppress +from datetime import datetime + +import pywikibot + +from pywikibot import async_request, page_put_queue + +from tests import patch, MagicMock +from tests.aspects import ( + unittest, DefaultSiteTestCase, DefaultDrySiteTestCase, +) +from tests.utils import entered_loop + + +class TestSiteInfo(DefaultSiteTestCase): + + """Test cases for Site metadata and capabilities.""" + + cached = True + + def test_siteinfo(self): + """Test the siteinfo property.""" + # general enteries + mysite = self.get_site() + self.assertIsInstance(mysite.siteinfo['timeoffset'], (int, float)) + self.assertTrue(-12 * 60 <= mysite.siteinfo['timeoffset'] <= +14 * 60) + self.assertEqual(mysite.siteinfo['timeoffset'] % 15, 0) + self.assertRegex(mysite.siteinfo['timezone'], + '([A-Z]{3,4}|[A-Z][a-z]+/[A-Z][a-z]+)') + self.assertIn(mysite.siteinfo['case'], ['first-letter', + 'case-sensitive']) + self.assertIsInstance( + datetime.strptime(mysite.siteinfo['time'], '%Y-%m-%dT%H:%M:%SZ'), + datetime) + self.assertEqual(re.findall(r'\$1', mysite.siteinfo['articlepath']), + ['$1']) + + def test_siteinfo_boolean(self): + """Test conversion of boolean properties from empty strings.""" + mysite = self.get_site() + self.assertIsInstance(mysite.siteinfo['titleconversion'], bool) + + self.assertIsInstance(mysite.namespaces[0].subpages, bool) + self.assertIsInstance(mysite.namespaces[0].content, bool) + + def test_properties_with_defaults(self): + """Test the siteinfo properties with defaults.""" + # This does not test that the defaults work correct, + # unless the default site is a version needing these defaults + # 'fileextensions' introduced in v1.15: + self.assertIsInstance(self.site.siteinfo.get('fileextensions'), list) + self.assertIn('fileextensions', self.site.siteinfo) + fileextensions = self.site.siteinfo.get('fileextensions') + self.assertIn({'ext': 'png'}, fileextensions) + # 'restrictions' introduced in v1.23: + mysite = self.site + self.assertIsInstance(mysite.siteinfo.get('restrictions'), dict) + self.assertIn('restrictions', mysite.siteinfo) + restrictions = self.site.siteinfo.get('restrictions') + self.assertIn('cascadinglevels', restrictions) + + def test_no_cache(self): + """Test siteinfo caching can be disabled.""" + if 'fileextensions' in self.site.siteinfo._cache: + del self.site.siteinfo._cache['fileextensions'] + self.site.siteinfo.get('fileextensions', cache=False) + self.assertNotIn('fileextensions', self.site.siteinfo) + + def test_not_exists(self): + """Test accessing a property not in siteinfo.""" + not_exists = 'this-property-does-not-exist' + mysite = self.site + self.assertRaises(KeyError, mysite.siteinfo.__getitem__, not_exists) + self.assertNotIn(not_exists, mysite.siteinfo) + self.assertIsEmpty(mysite.siteinfo.get(not_exists)) + self.assertFalse(entered_loop(mysite.siteinfo.get(not_exists))) + self.assertFalse( + entered_loop(mysite.siteinfo.get(not_exists).items())) + self.assertFalse( + entered_loop(mysite.siteinfo.get(not_exists).values())) + self.assertFalse(entered_loop(mysite.siteinfo.get(not_exists).keys())) + + +class TestSiteinfoDry(DefaultDrySiteTestCase): + + """Test Siteinfo in dry mode.""" + + def test_siteinfo_timestamps(self): + """Test that cache has the timestamp of CachedRequest.""" + site = self.get_site() + request_mock = MagicMock() + request_mock.submit = lambda: {'query': {'_prop': '_value'}} + request_mock._cachetime = '_cache_time' + with patch.object(site, '_request', return_value=request_mock): + siteinfo = pywikibot.site.Siteinfo(site) + result = siteinfo._get_siteinfo('_prop', False) + self.assertEqual(result, {'_prop': ('_value', '_cache_time')}) + + +class TestSiteinfoAsync(DefaultSiteTestCase): + + """Test asynchronous siteinfo fetch.""" + + def test_async_request(self): + """Test async request.""" + self.assertTrue(page_put_queue.empty()) + self.assertNotIn('statistics', self.site.siteinfo) + async_request(self.site.siteinfo.get, 'statistics') + page_put_queue.join() + self.assertIn('statistics', self.site.siteinfo) + + +if __name__ == '__main__': # pragma: no cover + with suppress(SystemExit): + unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/616320 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I8a87dc2bf0b178aa862448153dbffa3fcfe81f79 Gerrit-Change-Number: 616320 Gerrit-PatchSet: 16 Gerrit-Owner: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Dvorapa &lt;dvorapa(a)seznam.cz&gt; Gerrit-Reviewer: Matěj Suchánek &lt;matejsuchanek97(a)gmail.com&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: jenkins-bot Gerrit-CC: Huji &lt;huji.huji(a)gmail.com&gt; Gerrit-MessageType: merged