jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/453944 )
Change subject: pywikibot.site.Siteinfo: Fix the bug in cache_time when loading a CachedRequest
......................................................................
pywikibot.site.Siteinfo: Fix the bug in cache_time when loading a CachedRequest
Siteinfo._post_process:
- Make _post_process a static method (self was not used)
Siteinfo._get_siteinfo:
- Make the try block narrower. Only request.submit raises APIError.
- If the request was a CachedRequest, use the _cachetime attribute
as cache_time. This will resolve T202227.
Note that the _cachetime attribute is not UTC. In order to be
compatible we will need to change CachedRequest to always use
UTC time.
Siteinfo.get:
- The cache in the except clause was not used. Replace it with a `pass`.
api.CachedRequest:
- Always use UTC timestamps. (see the explanation of
Siteinfo._get_siteinfo)
api_tests.py, dry_api_tests.py, cache.py:
- Update code related to api.CachedRequest to use utcnow.
site_tests.py:
- Add a dry test for this fix. The result of this test used to
contain the value of utcnow() instead of '_cache_time'.
tests.__init__:
Import MagicMock and add it to __all__ so that it can be used more
conveniently.
Bug: T202227
Change-Id: Ic5fe36b3a2440c3e7fdf6837996d0b162d7643c7
---
M pywikibot/data/api.py
M pywikibot/site.py
M scripts/maintenance/cache.py
M tests/__init__.py
M tests/api_tests.py
M tests/dry_api_tests.py
M tests/site_tests.py
7 files changed, 48 insertions(+), 27 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 2acf7fb..9adc8f5 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -2389,7 +2389,7 @@
self._create_file_name())
def _expired(self, dt):
- return dt + self.expiry < datetime.datetime.now()
+ return dt + self.expiry < datetime.datetime.utcnow()
def _load_cache(self):
"""Load cache entry for request, if available.
@@ -2419,7 +2419,7 @@
def _write_cache(self, data):
"""Write data to self._cachefile_path()."""
- data = [self._uniquedescriptionstr(), data, datetime.datetime.now()]
+ data = (self._uniquedescriptionstr(), data, datetime.datetime.utcnow())
with open(self._cachefile_path(), 'wb') as f:
pickle.dump(data, f, protocol=config.pickle_protocol)
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 0adfbcc..0177c0e 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -1456,7 +1456,8 @@
else:
return pywikibot.tools.EMPTY_DEFAULT
- def _post_process(self, prop, data):
+ @staticmethod
+ def _post_process(prop, data):
"""Do some default handling of data. Directly modifies data."""
# Be careful with version tests inside this here as it might need to
# query this method to actually get the version number
@@ -1523,19 +1524,19 @@
if len(props) == 0:
raise ValueError('At least one property name must be provided.')
invalid_properties = []
+ request = self._site._request(
+ expiry=pywikibot.config.API_config_expiry
+ if expiry is False else expiry,
+ parameters={
+ 'action': 'query', 'meta': 'siteinfo', 'siprop': props,
+ }
+ )
+ # With 1.25wmf5 it'll require continue or rawcontinue. As we don't
+ # continue anyway we just always use continue.
+ request['continue'] = True
+ # warnings are handled later
+ request._warning_handler = warn_handler
try:
- request = self._site._request(
- expiry=pywikibot.config.API_config_expiry
- if expiry is False else expiry,
- parameters={
- 'action': 'query', 'meta': 'siteinfo', 'siprop': props,
- }
- )
- # With 1.25wmf5 it'll require continue or rawcontinue. As we don't
- # continue anyway we just always use continue.
- request['continue'] = True
- # warnings are handled later
- request._warning_handler = warn_handler
data = request.submit()
except api.APIError as e:
if e.code == 'siunknown_siprop':
@@ -1561,7 +1562,9 @@
pywikibot.log(u"Unable to get siprop(s) '{0}'".format(
u"', '".join(invalid_properties)))
if 'query' in data:
- cache_time = datetime.datetime.utcnow()
+ # If the request is a CachedRequest, use the _cachetime attr.
+ cache_time = getattr(
+ request, '_cachetime', None) or datetime.datetime.utcnow()
for prop in props:
if prop in data['query']:
self._post_process(prop, data['query'][prop])
@@ -1658,7 +1661,7 @@
try:
cached = self._get_cached(key)
except KeyError:
- cached = None
+ pass
else: # cached value available
# is a default value, but isn't accepted
if not cached[1] and not get_default:
diff --git a/scripts/maintenance/cache.py b/scripts/maintenance/cache.py
index 27752da..6eb9085 100755
--- a/scripts/maintenance/cache.py
+++ b/scripts/maintenance/cache.py
@@ -348,13 +348,13 @@
def older_than(entry, interval):
"""Find older entries."""
- if entry._cachetime + interval < datetime.datetime.now():
+ if entry._cachetime + interval < datetime.datetime.utcnow():
return entry
def newer_than(entry, interval):
"""Find newer entries."""
- if entry._cachetime + interval >= datetime.datetime.now():
+ if entry._cachetime + interval >= datetime.datetime.utcnow():
return entry
diff --git a/tests/__init__.py b/tests/__init__.py
index 650ccac..39a1651 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -7,8 +7,9 @@
#
from __future__ import absolute_import, print_function, unicode_literals
-__all__ = ('requests', 'unittest', 'TestRequest',
- 'patch_request', 'unpatch_request', 'mock', 'Mock', 'patch')
+__all__ = (
+ 'requests', 'unittest', 'TestRequest', 'patch_request', 'unpatch_request',
+ 'mock', 'Mock', 'MagicMock', 'patch')
import functools
import os
@@ -35,10 +36,10 @@
import unittest
try:
import unittest.mock as mock
- from unittest.mock import Mock, patch
+ from unittest.mock import MagicMock, Mock, patch
except ImportError:
import mock
- from mock import Mock, patch
+ from mock import MagicMock, Mock, patch
_root_dir = os.path.split(os.path.split(__file__)[0])[0]
diff --git a/tests/api_tests.py b/tests/api_tests.py
index 0a04e61..7e0d9b6 100644
--- a/tests/api_tests.py
+++ b/tests/api_tests.py
@@ -933,7 +933,7 @@
mysite = self.get_site()
# Run tests on a missing page unique to this test run so it can
# not be cached the first request, but will be cached after.
- now = datetime.datetime.now()
+ now = datetime.datetime.utcnow()
params = {'action': 'query',
'prop': 'info',
'titles': 'TestCachedRequest_test_internals ' + str(now),
diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py
index 2b2f206..b0ec916 100644
--- a/tests/dry_api_tests.py
+++ b/tests/dry_api_tests.py
@@ -73,8 +73,9 @@
def test_expired(self):
"""Test if the request is expired."""
- self.assertFalse(self.req._expired(datetime.datetime.now()))
- self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
+ self.assertFalse(self.req._expired(datetime.datetime.utcnow()))
+ self.assertTrue(self.req._expired(
+ datetime.datetime.utcnow() - datetime.timedelta(days=2)))
def test_parameter_types(self):
"""Test _uniquedescriptionstr is identical using different ways."""
diff --git a/tests/site_tests.py b/tests/site_tests.py
index 01ae4b0..cf7a9b4 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -34,7 +34,7 @@
UnicodeType as unicode,
)
-from tests import patch, unittest_print
+from tests import patch, unittest_print, MagicMock
from tests.aspects import (
unittest, TestCase, DeprecationTestCase,
TestCaseBase,
@@ -2414,6 +2414,22 @@
self.assertFalse(entered_loop(mysite.siteinfo.get(not_exists).keys()))
+class TestSiteinfoDry(DefaultDrySiteTestCase):
+
+ """Test Siteinfo in dry mode."""
+
+ def test_siteinfo_timestamps(self):
+ """Test that cache has the timestamp of CachedRequest."""
+ site = self.get_site()
+ request_mock = MagicMock()
+ request_mock.submit = lambda: {'query': {'_prop': '_value'}}
+ request_mock._cachetime = '_cache_time'
+ with patch.object(site, '_request', return_value=request_mock):
+ siteinfo = pywikibot.site.Siteinfo(site)
+ result = siteinfo._get_siteinfo('_prop', False)
+ self.assertEqual(result, {'_prop': ('_value', '_cache_time')})
+
+
class TestSiteinfoAsync(DefaultSiteTestCase):
"""Test asynchronous siteinfo fetch."""
--
To view, visit https://gerrit.wikimedia.org/r/453944
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic5fe36b3a2440c3e7fdf6837996d0b162d7643c7
Gerrit-Change-Number: 453944
Gerrit-PatchSet: 11
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot (75)
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/428066 )
Change subject: [IMPR] reduce code complexity of Request.submit
......................................................................
[IMPR] reduce code complexity of Request.submit
- Reduce cyclomatic complexity of submit from F72 to D30
and improve readability by splitting code in parts.
Change-Id: I45dda15d986ada4b4aff109538cd9739bff63c4b
---
M pywikibot/data/api.py
1 file changed, 276 insertions(+), 218 deletions(-)
Approvals:
Dalba: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index c305dda..bc0f826 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -1841,6 +1841,32 @@
submsg.set_payload(content)
return submsg
+ def _use_get(self):
+ """Verify whether 'get' is to be used."""
+ if (not config.enable_GET_without_SSL
+ and self.site.protocol() != 'https'
+ or self.site.is_oauth_token_available()): # T108182 workaround
+ use_get = False
+ elif self.use_get is None:
+ if self.action == 'query':
+ # for queries check the query module
+ modules = set()
+ for mod_type_name in ('list', 'prop', 'generator'):
+ modules.update(self._params.get(mod_type_name, []))
+ else:
+ modules = {self.action}
+ if modules:
+ self.site._paraminfo.fetch(modules)
+ use_get = all('mustbeposted' not in self.site._paraminfo[mod]
+ for mod in modules)
+ else:
+ # If modules is empty, just 'meta' was given, which doesn't
+ # require POSTs, and is required for ParamInfo
+ use_get = True
+ else:
+ use_get = self.use_get
+ return use_get
+
@classmethod
def _build_mime_request(cls, params, mime_params):
"""
@@ -1874,6 +1900,124 @@
headers = dict(container.items())
return headers, body
+ def _get_request_params(self, use_get, paramstring):
+ """Get request parameters."""
+ uri = self.site.apipath()
+ if self.mime:
+ (headers, body) = Request._build_mime_request(
+ self._encoded_items(), self.mime_params)
+ use_get = False # MIME requests require HTTP POST
+ else:
+ headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+ if (not self.site.maximum_GET_length()
+ or self.site.maximum_GET_length() < len(paramstring)):
+ use_get = False
+ if use_get:
+ uri = '{0}?{1}'.format(uri, paramstring)
+ body = None
+ else:
+ body = paramstring
+
+ pywikibot.debug('API request to {0} (uses get: {1}):\n'
+ 'Headers: {2!r}\nURI: {3!r}\nBody: {4!r}'
+ .format(self.site, use_get, headers, uri, body),
+ _logger)
+ return use_get, uri, body, headers
+
+ def _http_request(self, use_get, uri, body, headers, paramstring):
+ """Get or post a http request with exception handling.
+
+ @return: a tuple containing data from request and use_get value
+ @rtype: tuple
+ """
+ try:
+ data = http.request(
+ site=self.site, uri=uri,
+ method='GET' if use_get else 'POST',
+ body=body, headers=headers)
+ except Server504Error:
+ pywikibot.log('Caught HTTP 504 error; retrying')
+ except Server414Error:
+ if use_get:
+ pywikibot.log('Caught HTTP 414 error; retrying')
+ use_get = False
+ else:
+ pywikibot.warning('Caught HTTP 414 error, although not '
+ 'using GET.')
+ raise
+ except FatalServerError:
+ # This error is not going to be fixed by just waiting
+ pywikibot.error(traceback.format_exc())
+ raise
+ # TODO: what other exceptions can occur here?
+ except Exception:
+ # for any other error on the http request, wait and retry
+ pywikibot.error(traceback.format_exc())
+ pywikibot.log('{}, {}'.format(uri, paramstring))
+ else:
+ return data, use_get
+ self.wait()
+ return None, use_get
+
+ def _json_loads(self, data):
+ """Read source text and return a dict.
+
+ @param data: raw data string
+ @type data: str
+ @return: a data dict
+ @rtype: dict
+ @raises APIError: unknown action found
+ @raises APIError: unknown query result type
+ """
+ if not isinstance(data, unicode):
+ data = data.decode(self.site.encoding())
+ pywikibot.debug(('API response received from {}:\n'
+ .format(self.site)) + data, _logger)
+ if data.startswith('unknown_action'):
+ raise APIError(data[:14], data[16:])
+ try:
+ result = json.loads(data)
+ except ValueError:
+ # if the result isn't valid JSON, there must be a server
+ # problem. Wait a few seconds and try again
+ pywikibot.warning(
+ 'Non-JSON response received from server {}; '
+ 'the server may be down.'.format(self.site))
+ # there might also be an overflow, so try a smaller limit
+ for param in self._params:
+ if param.endswith('limit'):
+ # param values are stored a list of str
+ value = self._params[param][0]
+ if value.isdigit():
+ self._params[param] = [str(int(value) // 2)]
+ pywikibot.output('Set {} = {}'
+ .format(param, self._params[param]))
+ else:
+ if result and not isinstance(result, dict):
+ raise APIError('Unknown',
+ 'Unable to process query response of type {}.'
+ .format(type(result)),
+ data=result)
+ return result or {}
+ self.wait()
+ return None
+
+ def _userinfo_query(self, result):
+ """Handle userinfo query."""
+ if self.action == 'query' and 'userinfo' in result.get('query', ()):
+ # if we get passed userinfo in the query result, we can confirm
+ # that we are logged in as the correct user. If this is not the
+ # case, force a re-login.
+ username = result['query']['userinfo']['name']
+ if self.site.user() is not None and self.site.user() != username:
+ pywikibot.error(
+ "Logged in as '{actual}' instead of '{expected}'.\n"
+ 'Forcing re-login.'.format(actual=username,
+ expected=self.site.user()))
+ self.site._relogin()
+ return True
+ return False
+
def _handle_warnings(self, result):
if 'warnings' in result:
for mod, warning in result['warnings'].items():
@@ -1896,6 +2040,115 @@
pywikibot.warning('API warning ({}): {}'
.format(mod, single_warning))
+ def _logged_in(self, code):
+ """Check whether user is logged in.
+
+ Older wikis returned an error instead of a warning when the request
+ asked for too many values. If we get this error, assume we are not
+ logged in (we can't check this because the userinfo data is not
+ present) and force a re-login
+ """
+ if code.endswith('limit'):
+ message = 'Received API limit error.'
+
+ # If the user assertion failed, we're probably logged out as well.
+ elif code == 'assertuserfailed':
+ message = 'User assertion failed.'
+
+ # Lastly, the purge module require a POST if used as anonymous user,
+ # but we normally send a GET request. If the API tells us the request
+ # has to be POSTed, we're probably logged out.
+ elif code == 'mustbeposted' and self.action == 'purge':
+ message = "Received unexpected 'mustbeposted' error."
+
+ else:
+ return True
+
+ pywikibot.error(message + ' Forcing re-login.')
+ self.site._relogin()
+ return False
+
+ def _internal_api_error(self, code, error, result):
+ """Check for internal_api_error_ or readonly and retry.
+
+ @raises APIMWException: internal_api_error or readonly
+ """
+ IAE = 'internal_api_error_'
+ if not (code.startswith(IAE) or code == 'readonly'):
+ return False
+
+ # T154011
+ class_name = code if code == 'readonly' else code[len(IAE):]
+
+ del error['code'] # is added via class_name
+ e = APIMWException(class_name, **error)
+
+ # If the error key is in this table, it is probably a temporary
+ # problem, so we will retry the edit.
+ # TODO: T154011: 'ReadOnlyError' seems replaced by 'readonly'
+ retry = class_name in ['DBConnectionError', # T64974
+ 'DBQueryError', # T60158
+ 'ReadOnlyError', # T61227
+ 'readonly', # T154011
+ ]
+
+ pywikibot.error('Detected MediaWiki API exception {}{}'
+ .format(e, '; retrying' if retry else '; raising'))
+ # Due to bug T66958, Page's repr may return non ASCII bytes
+ # Get as bytes in PY2 and decode with the console encoding as
+ # the rest should be ASCII anyway.
+ param_repr = str(self._params)
+ if PY2:
+ param_repr = param_repr.decode(config.console_encoding)
+ pywikibot.log('MediaWiki exception {} details:\n'
+ ' query=\n{}\n'
+ ' response=\n{}'
+ .format(class_name,
+ pprint.pformat(param_repr),
+ result))
+ if not retry:
+ raise e
+
+ self.wait()
+ return True
+
+ def _bad_token(self, code):
+ """Check for bad token."""
+ if code != 'badtoken':
+ return False
+
+ user_tokens = self.site.tokens._tokens[self.site.user()]
+ # all token values mapped to their type
+ tokens = {token: t_type for t_type, token in user_tokens.items()}
+ # determine which tokens are bad
+ invalid_param = {name: tokens[param[0]]
+ for name, param in self._params.items()
+ if len(param) == 1 and param[0] in tokens}
+ # doesn't care about the cache so can directly load them
+ if invalid_param:
+ pywikibot.log(
+ 'Bad token error for {}. Tokens for "{}" used in request; '
+ 'invalidated them.'
+ .format(self.site.user(),
+ '", "'.join(sorted(set(invalid_param.values())))))
+ self.site.tokens.load_tokens(set(invalid_param.values()))
+ # fix parameters; lets hope that it doesn't mistake actual
+ # parameters as tokens
+ for name, t_type in invalid_param.items():
+ self[name] = self.site.tokens[t_type]
+ return True
+ else:
+ # otherwise couldn't find any … weird there is nothing what
+ # can be done here because it doesn't know which parameters
+ # to fix
+ pywikibot.log(
+ 'Bad token error for {} but no parameter is using a '
+ 'token. Current tokens: {}'
+ .format(self.site.user(),
+ ', '.join('{}: {}'.format(*e)
+ for e in user_tokens.items())))
+ return False
+
def submit(self):
"""
Submit a query and parse the response.
@@ -1904,143 +2157,34 @@
@rtype: dict
"""
self._add_defaults()
- if (not config.enable_GET_without_SSL and
- self.site.protocol() != 'https' or
- self.site.is_oauth_token_available()): # work around T108182
- use_get = False
- elif self.use_get is None:
- if self.action == 'query':
- # for queries check the query module
- modules = set()
- for mod_type_name in ('list', 'prop', 'generator'):
- modules.update(self._params.get(mod_type_name, []))
- else:
- modules = {self.action}
- if modules:
- self.site._paraminfo.fetch(modules)
- use_get = all('mustbeposted' not in self.site._paraminfo[mod]
- for mod in modules)
- else:
- # If modules is empty, just 'meta' was given, which doesn't
- # require POSTs, and is required for ParamInfo
- use_get = True
- else:
- use_get = self.use_get
+ use_get = self._use_get()
+
while True:
paramstring = self._http_param_string()
+
simulate = self._simulate(self.action)
if simulate:
return simulate
+
if self.throttle:
self.site.throttle(write=self.write)
else:
pywikibot.log(
"Submitting unthrottled action '{0}'.".format(self.action))
- uri = self.site.scriptpath() + "/api.php"
- try:
- if self.mime:
- (headers, body) = Request._build_mime_request(
- self._encoded_items(), self.mime_params)
- use_get = False # MIME requests require HTTP POST
- else:
- headers = {
- 'Content-Type': 'application/x-www-form-urlencoded'}
- if (not self.site.maximum_GET_length() or
- self.site.maximum_GET_length() < len(paramstring)):
- use_get = False
- if use_get:
- uri = '{0}?{1}'.format(uri, paramstring)
- body = None
- else:
- body = paramstring
- pywikibot.debug('API request to {0} (uses get: {1}):\n'
- 'Headers: {2!r}\nURI: {3!r}\n'
- 'Body: {4!r}'.format(self.site, use_get,
- headers, uri, body),
- _logger)
+ use_get, uri, body, headers = self._get_request_params(use_get,
+ paramstring)
+ rawdata, use_get = self._http_request(use_get, uri, body, headers,
+ paramstring)
+ if rawdata is None:
+ continue
- rawdata = http.request(
- site=self.site, uri=uri,
- method='GET' if use_get else 'POST',
- body=body, headers=headers)
- except Server504Error:
- pywikibot.log(u"Caught HTTP 504 error; retrying")
- self.wait()
+ result = self._json_loads(rawdata)
+ if result is None:
continue
- except Server414Error:
- if use_get:
- pywikibot.log('Caught HTTP 414 error; retrying')
- use_get = False
- self.wait()
- continue
- else:
- pywikibot.warning('Caught HTTP 414 error, although not '
- 'using GET.')
- raise
- except FatalServerError:
- # This error is not going to be fixed by just waiting
- pywikibot.error(traceback.format_exc())
- raise
- # TODO: what other exceptions can occur here?
- except Exception:
- # for any other error on the http request, wait and retry
- pywikibot.error(traceback.format_exc())
- pywikibot.log(u"%s, %s" % (uri, paramstring))
- self.wait()
- continue
- if not isinstance(rawdata, unicode):
- rawdata = rawdata.decode(self.site.encoding())
- pywikibot.debug((u"API response received from %s:\n" % self.site) +
- rawdata, _logger)
- if rawdata.startswith(u"unknown_action"):
- raise APIError(rawdata[:14], rawdata[16:])
- try:
- result = json.loads(rawdata)
- except ValueError:
- # if the result isn't valid JSON, there must be a server
- # problem. Wait a few seconds and try again
- pywikibot.warning(
- 'Non-JSON response received from server {}; '
- 'the server may be down.'.format(self.site))
- # there might also be an overflow, so try a smaller limit
- for param in self._params:
- if param.endswith("limit"):
- # param values are stored a list of str
- value = self._params[param][0]
- try:
- self._params[param] = [str(int(value) // 2)]
- pywikibot.output(u"Set %s = %s"
- % (param, self._params[param]))
- except ValueError:
- pass
- self.wait()
- continue
- if not result:
- result = {}
- if not isinstance(result, dict):
- raise APIError("Unknown",
- "Unable to process query response of type %s."
- % type(result),
- data=result)
- if self.action == 'query' \
- and 'userinfo' in result.get('query', ()):
- # if we get passed userinfo in the query result, we can confirm
- # that we are logged in as the correct user. If this is not the
- # case, force a re-login.
- username = result['query']['userinfo']['name']
- if self.site.user() is not None \
- and self.site.user() != username:
- pywikibot.error(
- "Logged in as '{actual}' instead of '{expected}'. "
- "Forcing re-login.".format(
- actual=username,
- expected=self.site.user()
- )
- )
- self.site._relogin()
- continue
+ if self._userinfo_query(result):
+ continue
self._handle_warnings(result)
@@ -2062,29 +2206,7 @@
code = result['error'].setdefault('code', 'Unknown')
info = result['error'].setdefault('info', None)
- # Older wikis returned an error instead of a warning when
- # the request asked for too many values. If we get this
- # error, assume we are not logged in (we can't check this
- # because the userinfo data is not present) and force
- # a re-login
- if code.endswith('limit'):
- pywikibot.error("Received API limit error. Forcing re-login")
- self.site._relogin()
- continue
-
- # If the user assertion failed, we're probably logged out as well.
- if code == 'assertuserfailed':
- pywikibot.error("User assertion failed. Forcing re-login.")
- self.site._relogin()
- continue
-
- # Lastly, the purge module require a POST if used as anonymous
- # user, but we normally send a GET request. If the API tells us the
- # request has to be POSTed, we're probably logged out.
- if code == 'mustbeposted' and self.action == 'purge':
- pywikibot.error("Received unexpected 'mustbeposted' error. "
- "Forcing re-login.")
- self.site._relogin()
+ if not self._logged_in(code):
continue
if code == "maxlag":
@@ -2104,45 +2226,8 @@
pywikibot.warning('API error %s: %s' % (code, info))
- if code.startswith('internal_api_error_') or code == 'readonly':
- if code == 'readonly': # T154011
- class_name = code
- else:
- class_name = code[len(u'internal_api_error_'):]
-
- del error['code'] # is added via class_name
- e = APIMWException(class_name, **error)
-
- # If the error key is in this table, it is probably a temporary problem,
- # so we will retry the edit.
- # TODO: T154011: 'ReadOnlyError' seems replaced by 'readonly'
- retry = class_name in ['DBConnectionError', # T64974
- 'DBQueryError', # T60158
- 'ReadOnlyError', # T61227
- 'readonly', # T154011
- ]
-
- pywikibot.error("Detected MediaWiki API exception %s%s"
- % (e,
- "; retrying" if retry else "; raising"))
- # Due to bug T66958, Page's repr may return non ASCII bytes
- # Get as bytes in PY2 and decode with the console encoding as
- # the rest should be ASCII anyway.
- param_repr = str(self._params)
- if PY2:
- param_repr = param_repr.decode(config.console_encoding)
- pywikibot.log(u"MediaWiki exception %s details:\n"
- u" query=\n%s\n"
- u" response=\n%s"
- % (class_name,
- pprint.pformat(param_repr),
- result))
-
- if retry:
- self.wait()
- continue
-
- raise e
+ if self._internal_api_error(code, error, result):
+ continue
# Phab. tickets T48535, T64126, T68494, T68619
if code == "failed-save" and \
@@ -2150,43 +2235,15 @@
self._is_wikibase_error_retryable(result["error"]):
self.wait()
continue
+
# If readapidenied is returned try to login
if code == 'readapidenied' and self.site._loginstatus in (-3, -1):
self.site.login()
continue
- if code == 'badtoken':
- user_tokens = self.site.tokens._tokens[self.site.user()]
- # all token values mapped to their type
- tokens = {token: t_type
- for t_type, token in user_tokens.items()}
- # determine which tokens are bad
- invalid_param = {}
- for name, param in self._params.items():
- if len(param) == 1 and param[0] in tokens:
- invalid_param[name] = tokens[param[0]]
- # doesn't care about the cache so can directly load them
- if invalid_param:
- pywikibot.log(
- u'Bad token error for {0}. Tokens for "{1}" used in '
- u'request; invalidated them.'.format(
- self.site.user(),
- '", "'.join(sorted(set(invalid_param.values())))))
- self.site.tokens.load_tokens(set(invalid_param.values()))
- # fix parameters; lets hope that it doesn't mistake actual
- # parameters as tokens
- for name, t_type in invalid_param.items():
- self[name] = self.site.tokens[t_type]
- continue
- else:
- # otherwise couldn't find any … weird there is nothing what
- # can be done here because it doesn't know which parameters
- # to fix
- pywikibot.log(
- u'Bad token error for {0} but no parameter is using a '
- u'token. Current tokens: {1}'.format(
- self.site.user(),
- ', '.join('{0}: {1}'.format(*e)
- for e in user_tokens.items())))
+
+ if self._bad_token(code):
+ continue
+
if 'mwoauth-invalid-authorization' in code:
if 'Nonce already used' in info:
pywikibot.error(
@@ -2198,6 +2255,7 @@
if code == 'cirrussearch-too-busy-error': # T170647
self.wait()
continue
+
# raise error
try:
# Due to bug T66958, Page's repr may return non ASCII bytes
--
To view, visit https://gerrit.wikimedia.org/r/428066
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: I45dda15d986ada4b4aff109538cd9739bff63c4b
Gerrit-Change-Number: 428066
Gerrit-PatchSet: 5
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Framawiki <framawiki(a)tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot (75)
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/451824 )
Change subject: Use a key for filter_unique where appropriate
......................................................................
Use a key for filter_unique where appropriate
filter_unique can potentially consume a lot of memory since it stores
returned objects in a set if no `key` is provided. (T199615)
This patch adds an appropriate `key` to filter_unique calls wherever
the aforementioned issue might be a problem.
filter_unique has several usages:
- tools_tests.py:
We don't have any memory issues here.
- site.py:
No change is needed. The only usage of filter_unique is to filter
unique pageids which do not consume much memory.
- pagegenerators.py, scripts/template.py, scripts/nowcommons.py:
filter_unique is applied to page generators. Use
'{}:{}:{}'.format(*page._cmpkey()) as the key.
This patch-set is being proposed as an alternative to [1] that avoids
making `hash` the default key of `filter_unique`.
[1]: https://gerrit.wikimedia.org/r/c/pywikibot/core/+/445854/
Bug: T199615
Change-Id: Ia701e9f8bece713174ea4bccd104c0ae25ff5c1d
---
M pywikibot/pagegenerators.py
M scripts/nowcommons.py
M scripts/template.py
3 files changed, 9 insertions(+), 10 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 898237c..29e058a 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -29,6 +29,7 @@
import sys
from datetime import timedelta
+from functools import partial
from warnings import warn
import pywikibot
@@ -403,13 +404,12 @@
"""
# This is the function that will be used to de-duplicate iterators.
- # See the documentation in L{pywikibot.tools.filter_unique} for reasons
- # why this should be changed to improve space and time of execution.
- _filter_unique = staticmethod(filter_unique)
+ _filter_unique = staticmethod(partial(
+ filter_unique, key=lambda p: '{}:{}:{}'.format(*p._cmpkey())))
# The seen list can not yet be shared at present, due to `intersect` mode
# not being known until after all generators have been created.
# When not in intersect mode, _filter_unique could be:
- # functools.partial(filter_unique, container=global_seen_list)
+ # functools.partial(filter_unique, container=global_seen_list, key=...)
def __init__(self, site=None, positional_arg_name=None):
"""
@@ -1619,7 +1619,6 @@
@type namespaces: list of int
@param site: Site for generator results.
@type site: L{pywikibot.site.BaseSite}
-
"""
if site is None:
site = pywikibot.Site()
@@ -1630,9 +1629,9 @@
.format(user.username, site))
return _filter_unique(
- contrib[0]
- for contrib in user.contributions(namespaces=namespaces, total=total)
- )
+ (contrib[0] for contrib in user.contributions(
+ namespaces=namespaces, total=total)),
+ key=lambda p: '{}:{}:{}'.format(*p._cmpkey()))
def NamespaceFilterPageGenerator(generator, namespaces, site=None):
diff --git a/scripts/nowcommons.py b/scripts/nowcommons.py
index a82f52a..b29d7df 100755
--- a/scripts/nowcommons.py
+++ b/scripts/nowcommons.py
@@ -222,7 +222,7 @@
only_template_inclusion=True)
for t in self.nc_templates)
gen = chain(*gens)
- gen = filter_unique(gen)
+ gen = filter_unique(gen, key=lambda p: '{}:{}:{}'.format(*p._cmpkey()))
gen = pg.PreloadingGenerator(gen)
return gen
diff --git a/scripts/template.py b/scripts/template.py
index 570e23c..0a729b9 100755
--- a/scripts/template.py
+++ b/scripts/template.py
@@ -359,7 +359,7 @@
for t in old_templates
)
gen = chain(*gens)
- gen = filter_unique(gen)
+ gen = filter_unique(gen, key=lambda p: '{}:{}:{}'.format(*p._cmpkey()))
if user:
gen = pagegenerators.UserEditFilterGenerator(gen, user, timestamp,
skip,
--
To view, visit https://gerrit.wikimedia.org/r/451824
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia701e9f8bece713174ea4bccd104c0ae25ff5c1d
Gerrit-Change-Number: 451824
Gerrit-PatchSet: 4
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot (75)
Gerrit-CC: Zhuyifei1999 <zhuyifei1999(a)gmail.com>