jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/625361 ) Change subject: [4.0] Use Python 3 code in api.py ...................................................................... [4.0] Use Python 3 code in api.py Change-Id: If95de958d66cf46e30e96846537b4ac44aac2963 --- M pywikibot/data/api.py 1 file changed, 85 insertions(+), 152 deletions(-) Approvals: Matěj Suchánek: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index fd72de1..516731d 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -16,11 +16,13 @@ import traceback from collections.abc import Container, MutableMapping, Sized +from contextlib import suppress from email.generator import BytesGenerator from email.mime.multipart import MIMEMultipart as MIMEMultipartOrig from email.mime.nonmultipart import MIMENonMultipart from inspect import getfullargspec from io import BytesIO +from typing import Optional, Set, Tuple, Union from warnings import warn from urllib.parse import urlencode, unquote @@ -74,14 +76,14 @@ def __init__(self, *args, **kwargs): """Initializer.""" - super(CTEBinaryBytesGenerator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._writeBody = self._write_body def _write_body(self, msg): if msg['content-transfer-encoding'] == 'binary': self._fp.write(msg.get_payload(decode=True)) else: - super(CTEBinaryBytesGenerator, self)._handle_text(msg) + super()._handle_text(msg) class CTEBinaryMIMEMultipart(MIMEMultipartOrig): @@ -133,18 +135,18 @@ """Upload failed with a warning message (passed as the argument).""" - def __init__(self, code, message, file_key=None, offset=0): + def __init__(self, code, message, + file_key: Optional[str] = None, + offset: Union[int, bool] = 0): """ Create a new UploadWarning instance. @param filekey: The filekey of the uploaded file to reuse it later. If no key is known or it is an incomplete file it may be None. - @type filekey: str or None @param offset: The starting offset for a chunked upload. Is False when there is no offset. - @type offset: int or bool """ - super(UploadWarning, self).__init__(code, message) + super().__init__(code, message) self.file_key = file_key self.offset = offset @@ -162,7 +164,7 @@ """Save error dict returned by MW API.""" self.mediawiki_exception_class_name = mediawiki_exception_class_name code = 'internal_api_error_' + mediawiki_exception_class_name - super(APIMWException, self).__init__(code, info, **kwargs) + super().__init__(code, info, **kwargs) class ParamInfo(Sized, Container): @@ -351,17 +353,16 @@ } @staticmethod - def _modules_to_set(modules): + def _modules_to_set(modules) -> set: """Return modules as a set. - @type modules: iterable or basestring - @rtype: set + @type modules: iterable or str """ if isinstance(modules, str): return set(modules.split('|')) return set(modules) - def fetch(self, modules): + def fetch(self, modules) -> None: """ Fetch paraminfo for multiple modules. @@ -370,7 +371,6 @@ @param modules: API modules to load @type modules: iterable or str - @rtype: NoneType """ if 'main' not in self._paraminfo: # The first request should be 'paraminfo', so that @@ -391,13 +391,11 @@ self._fetch(modules) - def _fetch(self, modules): + def _fetch(self, modules: set) -> None: """ Fetch paraminfo for multiple modules without initializing beforehand. @param modules: API modules to load and which haven't been loaded yet. - @type modules: set - @rtype: NoneType """ def module_generator(): """A generator yielding batches of modules.""" @@ -553,7 +551,7 @@ assert param['name'] == 'generator' and \ submodules >= set(param['type']) - def _normalize_modules(self, modules): + def _normalize_modules(self, modules) -> set: """Add query+ to any query module name not also in action modules.""" # Users will supply the wrong type, and expect it to work. modules = self._modules_to_set(modules) @@ -566,14 +564,13 @@ else mod for mod in modules} - def normalize_modules(self, modules): + def normalize_modules(self, modules) -> set: """ Convert the modules into module paths. Add query+ to any query module name not also in action modules. @return: The modules converted into a module paths - @rtype: set """ self._init() return self._normalize_modules(modules) @@ -657,7 +654,7 @@ else: raise KeyError(key) - def __contains__(self, key): + def __contains__(self, key) -> bool: """Return whether the key is valid.""" try: self[key] @@ -665,22 +662,19 @@ except KeyError: return False - def __len__(self): + def __len__(self) -> int: """Return number of cached modules.""" return len(self._paraminfo) - def parameter(self, module, param_name): + def parameter(self, module: str, param_name: str) -> Optional[dict]: """ Get details about one modules parameter. Returns None if the parameter does not exist. @param module: API module name - @type module: str @param param_name: parameter name in the module - @type param_name: str @return: metadata that describes how the parameter may be used - @rtype: dict or None """ # TODO: the 'description' field of each parameter is not in the default # output of v1.25, and can't removed from previous API versions. @@ -711,7 +705,7 @@ @property @deprecated('submodules() or module_paths', since='20150715') - def modules(self): + def modules(self) -> Set[str]: """ Set of all main and query modules without path prefixes. @@ -719,7 +713,6 @@ the action modules and query modules). @return: module names - @rtype: set of str """ return self.action_modules | self.query_modules @@ -751,16 +744,13 @@ """Set of all query module names without query+ path prefix.""" return self.submodules('query') - def submodules(self, name, path=False): + def submodules(self, name: str, path: bool = False) -> set: """ Set of all submodules. @param name: The name of the parent module. - @type name: str @param path: Whether the path and not the name is returned. - @type path: bool @return: The names or paths of the submodules. - @rtype: set """ if name not in self._modules: self.fetch([name]) @@ -772,7 +762,7 @@ @staticmethod def _prefix_submodules(modules, prefix): """Prefix submodules with path.""" - return {'{0}+{1}'.format(prefix, mod) for mod in modules} + return {'{}+{}'.format(prefix, mod) for mod in modules} @property @deprecated('prefix_map', since='20150715') @@ -800,7 +790,7 @@ if prefix} return self._prefix_map.copy() - def attributes(self, attribute, modules=None): + def attributes(self, attribute: str, modules: Optional[set] = None): """ Mapping of modules with an attribute to the attribute value. @@ -808,10 +798,8 @@ attribute is empty or set to False. @param attribute: attribute name - @type attribute: basestring @param modules: modules to include. If None (default), it'll load all modules including all submodules using the paths. - @type modules: set or None @rtype: dict using modules as keys """ if modules is None: @@ -822,17 +810,16 @@ for mod in modules if attribute in self[mod]} @deprecated('attributes', since='20150715') - def module_attribute_map(self, attribute, modules=None): + def module_attribute_map(self, attribute: str, + modules: Optional[set] = None): """ Mapping of modules with an attribute to the attribute value. @param attribute: attribute name - @type attribute: basestring @param modules: modules to include. If None (default) it'll load all action and query modules using the module names. It only uses the path for query modules which have the same name as an action module. - @type modules: set @rtype: dict using modules as keys """ if modules is None: @@ -868,22 +855,22 @@ None and after setting it, any site (even None) will fail. """ - def __init__(self, site=None, module=None, param=None, dict=None): + def __init__(self, site=None, + module: Optional[str] = None, + param: Optional[str] = None, + dict: Optional[dict] = None): """ Initializer. If a site is given, the module and param must be given too. @param site: The associated site - @type site: piwikibot.site.APISite + @type site: pywikibot.site.APISite or None @param module: The module name which is used by paraminfo. (Ignored when site is None) - @type module: str @param param: The parameter name inside the module. That parameter must have a 'type' entry. (Ignored when site is None) - @type param: str @param dict: The initializing dict which is used for L{from_dict}. - @type dict: dict """ self._site_set = False self._enabled = set() @@ -892,9 +879,9 @@ if dict: self.from_dict(dict) - def _set_site(self, site, module, param, clear_invalid=False): - """ - Set the site and valid names. + def _set_site(self, site, module: str, param: str, + clear_invalid: bool = False): + """Set the site and valid names. As soon as the site has been not None, any subsequent calls will fail, unless there had been invalid names and a KeyError was thrown. @@ -902,13 +889,10 @@ @param site: The associated site @type site: pywikibot.site.APISite @param module: The module name which is used by paraminfo. - @type module: str @param param: The parameter name inside the module. That parameter must have a 'type' entry. - @type param: str @param clear_invalid: Instead of throwing a KeyError, invalid names are silently removed from the options (disabled by default). - @type clear_invalid: bool """ if self._site_set: raise TypeError('The site can not be set multiple times.') @@ -997,7 +981,7 @@ else: raise ValueError('Invalid value "{0}"'.format(value)) - def __getitem__(self, name): + def __getitem__(self, name) -> Optional[bool]: """ Return whether the option is enabled. @@ -1005,7 +989,6 @@ Otherwise it returns None. If the site has been set it raises a KeyError if the name is invalid. Otherwise it might return a value even though the name might be invalid. - @rtype: bool/None """ if name in self._enabled: return True @@ -1026,15 +1009,12 @@ def __iter__(self): """Iterate over each enabled and disabled option.""" - for enabled in self._enabled: - yield enabled - for disabled in self._disabled: - yield disabled + yield from self._enabled + yield from self._disabled def api_iter(self): """Iterate over each option as they appear in the URL.""" - for enabled in self._enabled: - yield enabled + yield from self._enabled for disabled in self._disabled: yield '!{0}'.format(disabled) @@ -1130,8 +1110,6 @@ so that when the API parameters are modified the changes can always be applied to the 'parameters' parameter. - @param parameters: The parameters used for the request to the API. - @type parameters: dict @param site: The Site to which the request will be submitted. If not supplied, uses the user's configured default Site. @param mime: If true, send in "multipart/form-data" format (default @@ -1150,6 +1128,8 @@ @param use_get: (optional) Use HTTP GET request if possible. If False it uses a POST request. If None, it'll try to determine via action=paraminfo if the action requires a POST. + @param parameters: The parameters used for the request to the API. + @type parameters: dict @param kwargs: The parameters used for the request to the API. """ if site is None: @@ -1277,7 +1257,7 @@ '"parameters" parameter.', DeprecationWarning, 3) @classmethod - def clean_kwargs(cls, kwargs): + def clean_kwargs(cls, kwargs: dict) -> dict: """ Convert keyword arguments into new parameters mode. @@ -1287,9 +1267,7 @@ added as a 'parameters' keyword. It will always create a shallow copy. @param kwargs: The original keyword arguments which is not modified. - @type kwargs: dict @return: The normalized keyword arguments. - @rtype: dict """ if 'expiry' in kwargs and kwargs['expiry'] is None: del kwargs['expiry'] @@ -1350,15 +1328,10 @@ """Implement dict interface.""" return self._params[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value): """Set MediaWiki API request parameter. - @param key: param key - - @type key: basestring - @param value: param value(s) - @type value: str in site encoding (string types may be a `|`-separated list) iterable, where items are converted to string @@ -1641,16 +1614,15 @@ return use_get @classmethod - def _build_mime_request(cls, params, mime_params): + def _build_mime_request(cls, params: dict, + mime_params) -> Tuple[dict, str]: """ Construct a MIME multipart form post. @param params: HTTP request params - @type params: dict @param mime_params: HTTP request parts which must be sent in the body @type mime_params: dict of (content, keytype, headers) @return: HTTP request headers and body - @rtype: tuple (dict, str) """ # construct a MIME message containing all API key/values container = MIMEMultipart(_subtype='form-data') @@ -1694,11 +1666,10 @@ _logger) return use_get, uri, body, headers - def _http_request(self, use_get, uri, body, headers, paramstring): + def _http_request(self, use_get, uri, body, headers, paramstring) -> tuple: """Get or post a http request with exception handling. @return: a tuple containing data from request and use_get value - @rtype: tuple """ try: data = http.request( @@ -1729,13 +1700,11 @@ self.wait() return None, use_get - def _json_loads(self, data): + def _json_loads(self, data: str) -> dict: """Read source text and return a dict. @param data: raw data string - @type data: str @return: a data dict - @rtype: dict @raises APIError: unknown action found @raises APIError: unknown query result type """ @@ -1899,7 +1868,7 @@ 'No rate limit found for action {}'.format(self.action)) self.wait(delay) - def _bad_token(self, code): + def _bad_token(self, code) -> bool: """Check for bad token.""" if (code != 'badtoken' or self.site._loginstatus == pywikibot.site.LoginStatus.IN_PROGRESS): @@ -1940,12 +1909,11 @@ for e in user_tokens.items()))) return False - def submit(self): + def submit(self) -> dict: """ Submit a query and parse the response. @return: a dict containing data retrieved from api.php - @rtype: dict """ self._add_defaults() use_get = self._use_get() @@ -2109,7 +2077,7 @@ @param expiry: either a number of days or a datetime.timedelta object """ assert expiry is not None - super(CachedRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if not isinstance(expiry, datetime.timedelta): expiry = datetime.timedelta(expiry) self.expiry = min(expiry, datetime.timedelta(config.API_config_expiry)) @@ -2122,14 +2090,13 @@ raise NotImplementedError('CachedRequest cannot be created simply.') @classmethod - def _get_cache_dir(cls): + def _get_cache_dir(cls) -> str: """ Return the base directory path for cache entries. The directory will be created if it does not already exist. @return: base directory path for cache entries - @rtype: basestring """ path = os.path.join(pywikibot.config2.base_dir, 'apicache-py{0:d}'.format(PYTHON_VERSION[0])) @@ -2138,33 +2105,24 @@ return path @staticmethod - def _make_dir(dir): - """ - Create directory if it does not exist already. + def _make_dir(dir_name: str) -> str: + """Create directory if it does not exist already. - The directory name (dir) is returned unmodified. + The directory name (dir_name) is returned unmodified. - @param dir: directory path - @type dir: basestring - + @param dir_name: directory path @return: directory name - @rtype: basestring """ - try: - os.makedirs(dir) - except OSError: - # directory already exists - pass - return dir + with suppress(OSError): # directory already exists + os.makedirs(dir_name) + return dir_name - def _uniquedescriptionstr(self): + def _uniquedescriptionstr(self) -> str: """Return unique description for the cache entry. If this is modified, please also update scripts/maintenance/cache.py to support the new key and all previous keys. - - @rtype: str """ login_status = self.site._loginstatus @@ -2199,11 +2157,10 @@ def _expired(self, dt): return dt + self.expiry < datetime.datetime.utcnow() - def _load_cache(self): + def _load_cache(self) -> bool: """Load cache entry for request, if available. @return: Whether the request was loaded from the cache - @rtype: bool """ self._add_defaults() try: @@ -2214,9 +2171,9 @@ if self._expired(self._cachetime): self._data = None return False - pywikibot.debug('%s: cache hit (%s) for API request: %s' - % (self.__class__.__name__, filename, uniquedescr), - _logger) + pywikibot.debug('{}: cache hit ({}) for API request: {}' + .format(self.__class__.__name__, filename, + uniquedescr), _logger) return True except IOError: # file not found @@ -2235,14 +2192,14 @@ """Submit cached request.""" cached_available = self._load_cache() if not cached_available: - self._data = super(CachedRequest, self).submit() + self._data = super().submit() self._write_cache(self._data) else: self._handle_warnings(self._data) return self._data -class _RequestWrapper(object): +class _RequestWrapper: """A wrapper class to handle the usage of the C{parameters} parameter.""" @@ -2271,8 +2228,8 @@ after iterating that many values. """ - def __init__(self, action, continue_name='continue', limit_name='limit', - data_name='data', **kwargs): + def __init__(self, action: str, continue_name: str = 'continue', + limit_name: str = 'limit', data_name: str = 'data', **kwargs): """ Initialize an APIGenerator object. @@ -2280,13 +2237,9 @@ documentation for values. @param action: API action name. - @type action: str @param continue_name: Name of the continue API parameter. - @type continue_name: str @param limit_name: Name of the limit API parameter. - @type limit_name: str @param data_name: Name of the data in API response. - @type data_name: str """ kwargs = self._clean_kwargs(kwargs, action=action) @@ -2303,7 +2256,7 @@ self.request = self.request_class(**kwargs) self.request[self.limit_name] = self.query_increment - def set_query_increment(self, value): + def set_query_increment(self, value: int): """ Set the maximum number of items to be retrieved per API query. @@ -2311,7 +2264,6 @@ @param value: The value of maximum number of items to be retrieved per API request to set. - @type value: int """ self.query_increment = int(value) self.request[self.limit_name] = self.query_increment @@ -2319,7 +2271,7 @@ % (self.__class__.__name__, self.query_increment), _logger) - def set_maximum_items(self, value): + def set_maximum_items(self, value: Union[int, str, None]): """ Set the maximum number of items to be retrieved from the wiki. @@ -2328,7 +2280,6 @@ @param value: The value of maximum number of items to be retrieved in total to set. Ignores None value. - @type value: int or str or None """ if value is not None and int(value) > 0: self.limit = int(value) @@ -2566,7 +2517,6 @@ If not called, the default is to ask for "max" items and let the API decide how many to send. - """ limit = int(value) @@ -2579,7 +2529,7 @@ % (self.__class__.__name__, self.query_limit), _logger) - def set_maximum_items(self, value): + def set_maximum_items(self, value: Union[int, str, None]): """Set the maximum number of items to be retrieved from the wiki. If not called, most queries will continue as long as there is @@ -2592,7 +2542,6 @@ @param value: The value of maximum number of items to be retrieved in total to set. Ignores None value. - @type value: int or str or None """ if value is not None: self.limit = int(value) @@ -2612,7 +2561,7 @@ self.api_limit), _logger) - def support_namespace(self): + def support_namespace(self) -> bool: """Check if namespace is a supported parameter on this query. Note: this function will be removed when self.set_namespace() will @@ -2620,7 +2569,6 @@ See T196619. @return: True if yes, False otherwise - @rtype: bool """ assert(self.limited_module) # some modules do not have a prefix return bool( @@ -2631,9 +2579,9 @@ """Set a namespace filter on this query. @param namespaces: namespace identifiers to limit query results - @type namespaces: iterable of basestring or Namespace key, - or a single instance of those types. May be a '|' separated - list of namespace identifiers. An empty iterator clears any + @type namespaces: iterable of str or Namespace key, or a single + instance of those types. May be a '|' separated list of + namespace identifiers. An empty iterator clears any namespace restriction. @raises KeyError: a namespace identifier was not resolved @@ -2827,8 +2775,7 @@ else: self.normalized = {} try: - for result in self._extract_results(resultdata): - yield result + yield from self._extract_results(resultdata) except RuntimeError: return # self.resultkey in data in last request.submit() @@ -2873,7 +2820,7 @@ """ - def __init__(self, generator, g_content=False, **kwargs): + def __init__(self, generator: str, g_content=False, **kwargs): """ Initializer. @@ -2881,7 +2828,6 @@ action=query is assumed and generator is required. @param generator: the "generator=" type from api.php - @type generator: str @param g_content: if True, retrieve the contents of the current version of each Page (default False) @@ -2963,7 +2909,7 @@ """ - def __init__(self, prop, **kwargs): + def __init__(self, prop: str, **kwargs): """ Initializer. @@ -2971,8 +2917,6 @@ action=query is assumed and prop is required. @param prop: the "prop=" type from api.php - @type prop: str - """ kwargs = self._clean_kwargs(kwargs, prop=prop) QueryGenerator.__init__(self, **kwargs) @@ -2987,18 +2931,13 @@ def __iter__(self): """Yield results.""" self._previous_dicts = {} - for result in super(PropertyGenerator, self).__iter__(): - yield result - for result in self._previous_dicts.values(): - yield result + yield from super().__iter__() + yield from self._previous_dicts.values() def _extract_results(self, resultdata): """Yield completed page_data of consecutive API requests.""" - for d in self._fully_retrieved_data_dicts(resultdata): - yield d - for data_dict in super(PropertyGenerator, self)._extract_results( - resultdata - ): + yield from self._fully_retrieved_data_dicts(resultdata) + for data_dict in super()._extract_results(resultdata): if 'title' in data_dict: d = self._previous_dicts.setdefault(data_dict['title'], data_dict) @@ -3046,7 +2985,7 @@ """ - def __init__(self, listaction, **kwargs): + def __init__(self, listaction: str, **kwargs): """ Initializer. @@ -3054,8 +2993,6 @@ action=query is assumed and listaction is required. @param listaction: the "list=" type from api.php - @type listaction: str - """ kwargs = self._clean_kwargs(kwargs, list=listaction) QueryGenerator.__init__(self, **kwargs) @@ -3116,7 +3053,6 @@ future. @return: empty string if successful, throws exception on failure - """ if hasattr(self, '_waituntil'): if datetime.datetime.now() < self._waituntil: @@ -3220,13 +3156,12 @@ """Ignore data; cookies are set by threadedhttp module.""" http.cookie_jar.save() - def get_login_token(self): + def get_login_token(self) -> str: """Fetch login token from action=query&meta=tokens. Requires MediaWiki >= 1.27. @return: login token - @rtype: str """ if self.site.mw_version < '1.27': raise NotImplementedError('The method get_login_token() requires ' @@ -3239,7 +3174,7 @@ return login_token_result['query']['tokens'].get('logintoken') -def encode_url(query): +def encode_url(query) -> str: """ Encode parameters to pass with a url. @@ -3251,7 +3186,6 @@ @param query: keys and values to be uncoded for passing with a url @type query: mapping object or a sequence of two-element tuples @return: encoded parameters with token parameters at the end - @rtype: str """ if hasattr(query, 'items'): query = list(query.items()) @@ -3263,7 +3197,7 @@ return urlencode(query) -def _update_pageid(page, pagedict): +def _update_pageid(page, pagedict: dict): """Update pageid.""" if 'pageid' in pagedict: page._pageid = int(pagedict['pageid']) @@ -3282,7 +3216,7 @@ .format(pagedict['title'])) -def _update_contentmodel(page, pagedict): +def _update_contentmodel(page, pagedict: dict): """Update page content model.""" page._contentmodel = pagedict.get('contentmodel') # can be None @@ -3293,7 +3227,7 @@ page._quality_text = pagedict['proofread']['quality_text'] -def _update_protection(page, pagedict): +def _update_protection(page, pagedict: dict): """Update page protection.""" if 'restrictiontypes' in pagedict: page._applicable_protections = set(pagedict['restrictiontypes']) @@ -3361,13 +3295,12 @@ page._coords = coords -def update_page(page, pagedict, props=[]): +def update_page(page, pagedict: dict, props=[]): """Update attributes of Page object page, based on query data in pagedict. @param page: object to be updated @type page: pywikibot.page.Page @param pagedict: the contents of a "page" element of a query response - @type pagedict: dict @param props: the property names which resulted in pagedict. If a missing value in pagedict can indicate both 'false' and 'not present' the property which would make the value present must be in the props -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/625361 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: If95de958d66cf46e30e96846537b4ac44aac2963 Gerrit-Change-Number: 625361 Gerrit-PatchSet: 7 Gerrit-Owner: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Matěj Suchánek &lt;matejsuchanek97(a)gmail.com&gt; Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged