jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631826 )
Change subject: [bugfix] print log header
......................................................................
[bugfix] print log header
pywikibot.comms.http does not have threads attribute anymore
- rewrite bot.writelogheader(). There is no reason for early return
- add init_routine to _inited_routines before init_routine is called
to prevent an infinite loop
Bug: T264235
Change-Id: Ia781ba848462889848309c77b90e6096b8fa8b41
---
M pywikibot/bot.py
M pywikibot/logging.py
2 files changed, 15 insertions(+), 20 deletions(-)
Approvals:
Mpaa: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 34eead8..8b5eb15 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -350,35 +350,29 @@
This may help the user to track errors or report bugs.
"""
- # If a http thread is not available, it's too early to print a header
- # that includes version information, which may need to query a server.
- # The http module can't be imported due to circular dependencies.
- http = sys.modules.get('pywikibot.comms.http', None)
- if not http or not hasattr(http, 'threads') or not len(http.threads):
- return
-
+ log('')
log('=== Pywikibot framework v{} -- Logging header ==='
.format(pywikibot.__version__))
# script call
- log('COMMAND: {0}'.format(sys.argv))
+ log('COMMAND: {}'.format(sys.argv))
# script call time stamp
- log('DATE: %s UTC' % str(datetime.datetime.utcnow()))
+ log('DATE: {} UTC'.format(datetime.datetime.utcnow()))
# new framework release/revision? (handle_args needs to be called first)
try:
- log('VERSION: %s' %
- version.getversion(online=config.log_pywiki_repo_version).strip())
+ log('VERSION: {}'.format(
+ version.getversion(online=config.log_pywiki_repo_version).strip()))
except version.ParseError:
exception()
# system
if hasattr(os, 'uname'):
- log('SYSTEM: {0}'.format(os.uname()))
+ log('SYSTEM: {}'.format(os.uname()))
# config file dir
- log('CONFIG FILE DIR: %s' % pywikibot.config2.base_dir)
+ log('CONFIG FILE DIR: {}'.format(pywikibot.config2.base_dir))
all_modules = sys.modules.keys()
@@ -401,9 +395,9 @@
'[{}]'.format(info.get('type', 'path unknown')))
info.setdefault('ver', '??')
if 'err' in info:
- log(' %(name)s: %(err)s' % info)
+ log(' {name}: {err}'.format_map(info))
else:
- log(' %(name)s (%(path)s) = %(ver)s' % info)
+ log(' {name} ({path}) = {ver}'.format_map(info))
# imported modules
log('MODULES:')
@@ -412,13 +406,13 @@
ver = version.get_module_version(module)
mtime = version.get_module_mtime(module)
if filename and ver and mtime:
- log(' {0} {1} {2}'
+ log(' {} {} {}'
.format(filename, ver[:7], mtime.isoformat(' ')))
if config.log_pywiki_repo_version:
- log('PYWIKI REPO VERSION: %s' % version.getversion_onlinerepo())
+ log('PYWIKI REPO VERSION: {}'.format(version.getversion_onlinerepo()))
- log('=== ' * 14)
+ log('=' * 57)
add_init_routine(init_handlers)
diff --git a/pywikibot/logging.py b/pywikibot/logging.py
index 83abad9..6951f62 100644
--- a/pywikibot/logging.py
+++ b/pywikibot/logging.py
@@ -28,9 +28,10 @@
def _init():
"""Init any routines which have not already been called."""
for init_routine in _init_routines:
- if init_routine not in _inited_routines:
- init_routine()
+ found = init_routine in _inited_routines # prevent infinite loop
_inited_routines.add(init_routine)
+ if not found:
+ init_routine()
# Clear the list of routines to be inited
_init_routines[:] = [] # the global variable is used with slice operator
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631826
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ia781ba848462889848309c77b90e6096b8fa8b41
Gerrit-Change-Number: 631826
Gerrit-PatchSet: 1
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/606413 )
Change subject: [IMPR] do not compare against None but use explicit "is None"
......................................................................
[IMPR] do not compare against None but use explicit "is None"
- do not compare against None but use explicit "is None"
- also use an explicit return at end of the function
if function have already a return value other than None
Change-Id: Ifb4646b601d2370471459cc564af1b3537c303f5
---
M pywikibot/comms/threadedhttp.py
1 file changed, 10 insertions(+), 14 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py
index 1f24e6b..b8c3e24 100644
--- a/pywikibot/comms/threadedhttp.py
+++ b/pywikibot/comms/threadedhttp.py
@@ -76,20 +76,17 @@
@property
def exception(self):
"""Get the exception, if any."""
- if isinstance(self.data, Exception):
- return self.data
+ return self.data if isinstance(self.data, Exception) else None
@property
def response_headers(self):
"""Return the response headers."""
- if not self.exception:
- return self.data.headers
+ return self.data.headers if not self.exception else None
@property
def raw(self):
"""Return the raw response body."""
- if not self.exception:
- return self.data.content
+ return self.data.content if not self.exception else None
@property
def parsed_uri(self):
@@ -109,8 +106,7 @@
@rtype: int
"""
- if not self.exception:
- return self.data.status_code
+ return self.data.status_code if not self.exception else None
@property
def header_encoding(self):
@@ -146,15 +142,15 @@
charset = 'latin1'
else:
charset = self.charset
+ lookup = codecs.lookup(charset) if charset else None
if (self.header_encoding
- and codecs.lookup(
- self.header_encoding) != (
- codecs.lookup(charset) if charset else None)):
+ and (lookup is None
+ or codecs.lookup(self.header_encoding) != lookup)):
if charset:
pywikibot.warning(
- 'Encoding "{0}" requested but "{1}" '
- 'received in the header.'.format(
- charset, self.header_encoding))
+ 'Encoding "{}" requested but "{}" '
+ 'received in the header.'
+ .format(charset, self.header_encoding))
try:
# TODO: Buffer decoded content, weakref does remove it too
# early (directly after this method)
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/606413
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ifb4646b601d2370471459cc564af1b3537c303f5
Gerrit-Change-Number: 606413
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631766 )
Change subject: Rename format parameter to content_format in flow.py
......................................................................
Rename format parameter to content_format in flow.py
Patch detached from Ifd003f061d63437c74f1f766d5c78bf69e83183a
Change-Id: Ibf9e6483378469d410d56050a1843552ccec6357
---
M pywikibot/flow.py
M pywikibot/site/__init__.py
M tests/flow_edit_tests.py
M tests/flow_tests.py
4 files changed, 137 insertions(+), 121 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 29c7579..016d66d 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -11,6 +11,7 @@
from pywikibot.exceptions import NoPage, UnknownExtension, LockedPage
from pywikibot.page import BasePage, User
+from pywikibot.tools import deprecate_arg
logger = logging.getLogger('pywiki.wiki.flow')
@@ -74,8 +75,11 @@
"""A Flow discussion board."""
- def _load(self, force=False):
- """Load and cache the Board's data, derived from its topic list."""
+ def _load(self, force: bool = False):
+ """Load and cache the Board's data, derived from its topic list.
+
+ @param force: Whether to force a reload if the data is already loaded
+ """
if not hasattr(self, '_data') or force:
self._data = self.site.load_board(self)
return self._data
@@ -95,34 +99,31 @@
new_params[key] = value
return new_params
- def topics(self, format='wikitext', limit=100, sort_by='newest',
- offset=None, offset_uuid='', reverse=False,
- include_offset=False, toc_only=False):
+ @deprecate_arg('format', 'content_format')
+ def topics(self, content_format: str = 'wikitext', limit: int = 100,
+ sort_by: str = 'newest', offset=None, offset_uuid: str = '',
+ reverse: bool = False, include_offset: bool = False,
+ toc_only: bool = False):
"""Load this board's topics.
- @param format: The content format to request the data in.
- @type format: str (either 'wikitext', 'html', or 'fixed-html')
+ @param content_format: The content format to request the data in;
+ must be either 'wikitext', 'html', or 'fixed-html'
@param limit: The number of topics to fetch in each request.
- @type limit: int
- @param sort_by: Algorithm to sort topics by.
- @type sort_by: str (either 'newest' or 'updated')
+ @param sort_by: Algorithm to sort topics by;
+ must be either 'newest' or 'updated'
@param offset: The timestamp to start at (when sortby is 'updated').
@type offset: Timestamp or equivalent str
@param offset_uuid: The UUID to start at (when sortby is 'newest').
- @type offset_uuid: str (in the form of a UUID)
@param reverse: Whether to reverse the topic ordering.
- @type reverse: bool
@param include_offset: Whether to include the offset topic.
- @type include_offset: bool
@param toc_only: Whether to only include information for the TOC.
- @type toc_only: bool
@return: A generator of this board's topics.
@rtype: generator of Topic objects
"""
- data = self.site.load_topiclist(self, format=format, limit=limit,
- sortby=sort_by, toconly=toc_only,
- offset=offset, offset_id=offset_uuid,
- reverse=reverse,
+ data = self.site.load_topiclist(self, content_format=content_format,
+ limit=limit, sortby=sort_by,
+ toconly=toc_only, offset=offset,
+ offset_id=offset_uuid, reverse=reverse,
include_offset=include_offset)
while data['roots']:
for root in data['roots']:
@@ -131,29 +132,33 @@
cont_args = self._parse_url(data['links']['pagination'])
data = self.site.load_topiclist(self, **cont_args)
- def new_topic(self, title, content, format='wikitext'):
+ @deprecate_arg('format', 'content_format')
+ def new_topic(self, title: str, content: str,
+ content_format: str = 'wikitext'):
"""Create and return a Topic object for a new topic on this Board.
@param title: The title of the new topic (must be in plaintext)
- @type title: str
@param content: The content of the topic's initial post
- @type content: str
- @param format: The content format of the value supplied for content
- @type format: str (either 'wikitext' or 'html')
+ @param content_format: The content format of the supplied content;
+ either 'wikitext' or 'html'
@return: The new topic
@rtype: Topic
"""
- return Topic.create_topic(self, title, content, format)
+ return Topic.create_topic(self, title, content, content_format)
class Topic(FlowPage):
"""A Flow discussion topic."""
- def _load(self, format='wikitext', force=False):
- """Load and cache the Topic's data."""
+ def _load(self, content_format: str = 'wikitext', force: bool = False):
+ """Load and cache the Topic's data.
+
+ @param content_format: The post format in which to load
+ @param force: Whether to force a reload if the data is already loaded
+ """
if not hasattr(self, '_data') or force:
- self._data = self.site.load_topic(self, format)
+ self._data = self.site.load_topic(self, content_format)
return self._data
def _reload(self):
@@ -161,21 +166,22 @@
self.root._load(load_from_topic=True)
@classmethod
- def create_topic(cls, board, title, content, format='wikitext'):
+ @deprecate_arg('format', 'content_format')
+ def create_topic(cls, board, title: str, content: str,
+ content_format: str = 'wikitext'):
"""Create and return a Topic object for a new topic on a Board.
@param board: The topic's parent board
@type board: Board
@param title: The title of the new topic (must be in plaintext)
- @type title: str
@param content: The content of the topic's initial post
- @type content: str
- @param format: The content format of the value supplied for content
- @type format: str (either 'wikitext' or 'html')
+ @param content_format: The content format of the supplied content;
+ either 'wikitext' or 'html'
@return: The new topic
@rtype: Topic
"""
- data = board.site.create_new_topic(board, title, content, format)
+ data = board.site.create_new_topic(board, title, content,
+ content_format)
return cls(board.site, data['topic-page'])
@classmethod
@@ -220,29 +226,29 @@
"""Whether this topic is moderated."""
return self.root._current_revision['isModerated']
- def replies(self, format='wikitext', force=False):
+ @deprecate_arg('format', 'content_format')
+ def replies(self, content_format: str = 'wikitext', force: bool = False):
"""A list of replies to this topic's root post.
- @param format: Content format to return contents in
- @type format: str ('wikitext', 'html', or 'fixed-html')
+ @param content_format: Content format to return contents in;
+ must be 'wikitext', 'html', or 'fixed-html'
@param force: Whether to reload from the API instead of using the cache
- @type force: bool
@return: The replies of this topic's root post
@rtype: list of Posts
"""
- return self.root.replies(format=format, force=force)
+ return self.root.replies(content_format=content_format, force=force)
- def reply(self, content, format='wikitext'):
+ @deprecate_arg('format', 'content_format')
+ def reply(self, content: str, content_format: str = 'wikitext'):
"""A convenience method to reply to this topic's root post.
@param content: The content of the new post
- @type content: str
- @param format: The format of the given content
- @type format: str ('wikitext' or 'html')
+ @param content_format: The format of the given content;
+ must be 'wikitext' or 'html')
@return: The new reply to this topic's root post
@rtype: Post
"""
- return self.root.reply(content, format)
+ return self.root.reply(content, content_format)
# Moderation
def lock(self, reason):
@@ -376,13 +382,17 @@
assert isinstance(content['content'], str)
self._content[content['format']] = content['content']
- def _load(self, format='wikitext', load_from_topic=False):
- """Load and cache the Post's data using the given content format."""
+ @deprecate_arg('format', 'content_format')
+ def _load(self, content_format='wikitext', load_from_topic: bool = False):
+ """Load and cache the Post's data using the given content format.
+
+ @param load_from_topic: Whether to load the post from the whole topic
+ """
if load_from_topic:
- data = self.page._load(format=format, force=True)
+ data = self.page._load(content_format=content_format, force=True)
else:
data = self.site.load_post_current_revision(self.page, self.uuid,
- format)
+ content_format)
self._set_data(data)
return self._current_revision
@@ -430,31 +440,30 @@
self._current_revision['creator']['name'])
return self._creator
- def get(self, format='wikitext', force=False):
+ @deprecate_arg('format', 'content_format')
+ def get(self, content_format: str = 'wikitext',
+ force: bool = False) -> str:
"""Return the contents of the post in the given format.
@param force: Whether to reload from the API instead of using the cache
- @type force: bool
- @param format: Content format to return contents in
- @type format: str
+ @param content_format: Content format to return contents in
@return: The contents of the post in the given content format
- @rtype: str
"""
- if format not in self._content or force:
- self._load(format)
- return self._content[format]
+ if content_format not in self._content or force:
+ self._load(content_format)
+ return self._content[content_format]
- def replies(self, format='wikitext', force=False):
+ @deprecate_arg('format', 'content_format')
+ def replies(self, content_format: str = 'wikitext', force: bool = False):
"""Return this post's replies.
- @param format: Content format to return contents in
- @type format: str ('wikitext', 'html', or 'fixed-html')
+ @param content_format: Content format to return contents in;
+ must be 'wikitext', 'html', or 'fixed-html'
@param force: Whether to reload from the API instead of using the cache
- @type force: bool
@return: This post's replies
@rtype: list of Posts
"""
- if format not in ('wikitext', 'html', 'fixed-html'):
+ if content_format not in ('wikitext', 'html', 'fixed-html'):
raise ValueError('Invalid content format.')
if hasattr(self, '_replies') and not force:
@@ -463,20 +472,20 @@
# load_from_topic workaround due to T106733
# (replies not returned by view-post)
if not hasattr(self, '_current_revision') or force:
- self._load(format, load_from_topic=True)
+ self._load(content_format, load_from_topic=True)
reply_uuids = self._current_revision['replies']
self._replies = [Post(self.page, uuid) for uuid in reply_uuids]
return self._replies
- def reply(self, content, format='wikitext'):
+ @deprecate_arg('format', 'content_format')
+ def reply(self, content: str, content_format: str = 'wikitext'):
"""Reply to this post.
@param content: The content of the new post
- @type content: str
- @param format: The format of the given content
- @type format: str ('wikitext' or 'html')
+ @param content_format: The format of the given content;
+ must be 'wikitext' or 'html'
@return: The new reply post
@rtype: Post
"""
@@ -491,7 +500,8 @@
if self.uuid == reply_to:
del self._current_revision
del self._replies
- data = self.site.reply_to_post(self.page, reply_to, content, format)
+ data = self.site.reply_to_post(self.page, reply_to, content,
+ content_format)
post = Post(self.page, data['post-id'])
return post
diff --git a/pywikibot/site/__init__.py b/pywikibot/site/__init__.py
index 1f6f804..2d481b2 100644
--- a/pywikibot/site/__init__.py
+++ b/pywikibot/site/__init__.py
@@ -6408,7 +6408,8 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topiclist(self, page, format='wikitext', limit=100,
+ @deprecate_arg('format', 'content_format')
+ def load_topiclist(self, page, content_format: str = 'wikitext', limit=100,
sortby='newest', toconly=False, offset=None,
offset_id=None, reverse=False, include_offset=False):
"""
@@ -6416,8 +6417,8 @@
@param page: A Flow board
@type page: Board
- @param format: The content format to request the data in.
- @type format: str (either 'wikitext', 'html', or 'fixed-html')
+ @param content_format: The content format to request the data in.
+ must be either 'wikitext', 'html', or 'fixed-html'
@param limit: The number of topics to fetch in each request.
@type limit: int
@param sortby: Algorithm to sort topics by.
@@ -6441,7 +6442,7 @@
params = {'action': 'flow', 'submodule': 'view-topiclist',
'page': page,
- 'vtlformat': format, 'vtlsortby': sortby,
+ 'vtlformat': content_format, 'vtlsortby': sortby,
'vtllimit': limit, 'vtloffset-dir': offset_dir,
'vtloffset': offset, 'vtloffset-id': offset_id,
'vtlinclude-offset': include_offset, 'vtltoconly': toconly}
@@ -6450,25 +6451,27 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topic(self, page, format):
+ @deprecate_arg('format', 'content_format')
+ def load_topic(self, page, content_format: str):
"""
Retrieve the data for a Flow topic.
@param page: A Flow topic
@type page: Topic
- @param format: The content format to request the data in.
- @type format: str (either 'wikitext', 'html', or 'fixed-html')
+ @param content_format: The content format to request the data in.
+ Must ne either 'wikitext', 'html', or 'fixed-html'
@return: A dict representing the topic's data.
@rtype: dict
"""
req = self._simple_request(action='flow', page=page,
submodule='view-topic',
- vtformat=format)
+ vtformat=content_format)
data = req.submit()
return data['flow']['view-topic']['result']['topic']
@need_extension('Flow')
- def load_post_current_revision(self, page, post_id, format):
+ @deprecate_arg('format', 'content_format')
+ def load_post_current_revision(self, page, post_id, content_format: str):
"""
Retrieve the data for a post to a Flow topic.
@@ -6476,20 +6479,21 @@
@type page: Topic
@param post_id: The UUID of the Post
@type post_id: str
- @param format: The content format used for the returned content
- @type format: str (either 'wikitext', 'html', or 'fixed-html')
+ @param content_format: The content format used for the returned
+ content; must be either 'wikitext', 'html', or 'fixed-html'
@return: A dict representing the post data for the given UUID.
@rtype: dict
"""
req = self._simple_request(action='flow', page=page,
submodule='view-post', vppostId=post_id,
- vpformat=format)
+ vpformat=content_format)
data = req.submit()
return data['flow']['view-post']['result']['topic']
@need_right('edit')
@need_extension('Flow')
- def create_new_topic(self, page, title, content, format):
+ @deprecate_arg('format', 'content_format')
+ def create_new_topic(self, page, title, content, content_format):
"""
Create a new topic on a Flow board.
@@ -6499,14 +6503,14 @@
@type title: str
@param content: The content of the topic's initial post
@type content: str
- @param format: The content format of the value supplied for content
- @type format: str (either 'wikitext' or 'html')
+ @param content_format: The content format of the supplied content
+ @type content_format: str (either 'wikitext' or 'html')
@return: The metadata of the new topic
@rtype: dict
"""
token = self.tokens['csrf']
params = {'action': 'flow', 'page': page, 'token': token,
- 'submodule': 'new-topic', 'ntformat': format,
+ 'submodule': 'new-topic', 'ntformat': content_format,
'nttopic': title, 'ntcontent': content}
req = self._request(parameters=params, use_get=False)
data = req.submit()
@@ -6514,24 +6518,23 @@
@need_right('edit')
@need_extension('Flow')
- def reply_to_post(self, page, reply_to_uuid, content, format):
+ @deprecate_arg('format', 'content_format')
+ def reply_to_post(self, page, reply_to_uuid: str, content: str,
+ content_format: str) -> dict:
"""Reply to a post on a Flow topic.
@param page: A Flow topic
@type page: Topic
@param reply_to_uuid: The UUID of the Post to create a reply to
- @type reply_to_uuid: str
@param content: The content of the reply
- @type content: str
- @param format: The content format used for the supplied content
- @type format: str (either 'wikitext' or 'html')
+ @param content_format: The content format used for the supplied
+ content; must be either 'wikitext' or 'html'
@return: Metadata returned by the API
- @rtype: dict
"""
token = self.tokens['csrf']
params = {'action': 'flow', 'page': page, 'token': token,
'submodule': 'reply', 'repreplyTo': reply_to_uuid,
- 'repcontent': content, 'repformat': format}
+ 'repcontent': content, 'repformat': content_format}
req = self._request(parameters=params, use_get=False)
data = req.submit()
return data['flow']['reply']['committed']['topic']
diff --git a/tests/flow_edit_tests.py b/tests/flow_edit_tests.py
index bdf2797..d2cf3ba 100644
--- a/tests/flow_edit_tests.py
+++ b/tests/flow_edit_tests.py
@@ -14,6 +14,9 @@
from tests import unittest
+MODERATION_REASON = 'Pywikibot test'
+
+
class TestFlowCreateTopic(TestCase):
"""Test the creation of Flow topics."""
@@ -28,9 +31,9 @@
"""Test creation of topic."""
content = 'If you can read this, the Flow code in Pywikibot works!'
board = Board(self.site, 'Talk:Pywikibot test')
- topic = board.new_topic('Pywikibot test', content, 'wikitext')
+ topic = board.new_topic(MODERATION_REASON, content, 'wikitext')
first_post = topic.replies()[0]
- wikitext = first_post.get(format='wikitext')
+ wikitext = first_post.get(content_format='wikitext')
self.assertIn('wikitext', first_post._content)
self.assertNotIn('html', first_post._content)
self.assertIsInstance(wikitext, str)
@@ -62,7 +65,7 @@
# Reply
reply_post = topic.reply(content, 'wikitext')
# Test content
- wikitext = reply_post.get(format='wikitext')
+ wikitext = reply_post.get(content_format='wikitext')
self.assertIn('wikitext', reply_post._content)
self.assertNotIn('html', reply_post._content)
self.assertIsInstance(wikitext, str)
@@ -82,7 +85,7 @@
# Reply
reply_post = topic_root.reply(content, 'wikitext')
# Test content
- wikitext = reply_post.get(format='wikitext')
+ wikitext = reply_post.get(content_format='wikitext')
self.assertIn('wikitext', reply_post._content)
self.assertNotIn('html', reply_post._content)
self.assertIsInstance(wikitext, str)
@@ -101,7 +104,7 @@
# Reply
reply_post = root_post.reply(content, 'wikitext')
# Test content
- wikitext = reply_post.get(format='wikitext')
+ wikitext = reply_post.get(content_format='wikitext')
self.assertIn('wikitext', reply_post._content)
self.assertNotIn('html', reply_post._content)
self.assertIsInstance(wikitext, str)
@@ -122,7 +125,7 @@
old_root_replies = topic_root.replies(force=True)[:]
first_reply_post = topic_root.reply(first_content, 'wikitext')
# Test first reply's content
- first_wikitext = first_reply_post.get(format='wikitext')
+ first_wikitext = first_reply_post.get(content_format='wikitext')
self.assertIn('wikitext', first_reply_post._content)
self.assertNotIn('html', first_reply_post._content)
self.assertIsInstance(first_wikitext, str)
@@ -137,7 +140,7 @@
second_reply_post = first_reply_post.reply(second_content,
'wikitext')
# Test nested reply's content
- second_wikitext = second_reply_post.get(format='wikitext')
+ second_wikitext = second_reply_post.get(content_format='wikitext')
self.assertIn('wikitext', second_reply_post._content)
self.assertNotIn('html', second_reply_post._content)
self.assertIsInstance(second_wikitext, str)
@@ -169,13 +172,13 @@
# Setup
topic = Topic(self.site, 'Topic:Sn12rdih4iducjsd')
if topic.is_locked:
- topic.unlock()
+ topic.unlock(MODERATION_REASON)
self.assertFalse(topic.is_locked)
# Lock topic
- topic.lock('Pywikibot test')
+ topic.lock(MODERATION_REASON)
self.assertTrue(topic.is_locked)
# Unlock topic
- topic.unlock('Pywikibot test')
+ topic.unlock(MODERATION_REASON)
self.assertFalse(topic.is_locked)
@@ -194,13 +197,13 @@
# Setup
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
if topic.is_moderated:
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
# Hide
- topic.hide('Pywikibot test')
+ topic.hide(MODERATION_REASON)
self.assertTrue(topic.is_moderated)
# Restore
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
def test_hide_post(self):
@@ -209,13 +212,13 @@
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
post = Post(topic, 'sq1qvoig1az8w7cd')
if post.is_moderated:
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
# Hide
- post.hide('Pywikibot test')
+ post.hide(MODERATION_REASON)
self.assertTrue(post.is_moderated)
# Restore
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
@@ -235,13 +238,13 @@
# Setup
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
if topic.is_moderated:
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
# Delete
- topic.delete_mod('Pywikibot test')
+ topic.delete_mod(MODERATION_REASON)
self.assertTrue(topic.is_moderated)
# Restore
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
def test_delete_post(self):
@@ -250,13 +253,13 @@
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
post = Post(topic, 'sq1qvoig1az8w7cd')
if post.is_moderated:
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
# Delete
- post.delete('Pywikibot test')
+ post.delete(MODERATION_REASON)
self.assertTrue(post.is_moderated)
# Restore
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
@@ -277,13 +280,13 @@
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
post = Post(topic, 'sq1qvoig1az8w7cd')
if post.is_moderated:
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
# Suppress
- post.suppress('Pywikibot test')
+ post.suppress(MODERATION_REASON)
self.assertTrue(post.is_moderated)
# Restore
- post.restore('Pywikibot test')
+ post.restore(MODERATION_REASON)
self.assertFalse(post.is_moderated)
def test_suppress_topic(self):
@@ -291,13 +294,13 @@
# Setup
topic = Topic(self.site, 'Topic:Sl4svodmrhzmpjjh')
if topic.is_moderated:
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
# Suppress
- topic.suppress('Pywikibot test')
+ topic.suppress(MODERATION_REASON)
self.assertTrue(topic.is_moderated)
# Restore
- topic.restore('Pywikibot test')
+ topic.restore(MODERATION_REASON)
self.assertFalse(topic.is_moderated)
diff --git a/tests/flow_tests.py b/tests/flow_tests.py
index df04850..49995c7 100644
--- a/tests/flow_tests.py
+++ b/tests/flow_tests.py
@@ -116,26 +116,26 @@
topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2')
post = Post(topic, 'sh6wgoagna97q0ia')
# Wikitext
- wikitext = post.get(format='wikitext')
+ wikitext = post.get(content_format='wikitext')
self.assertIn('wikitext', post._content)
self.assertNotIn('html', post._content)
self.assertIsInstance(wikitext, str)
self.assertNotEqual(wikitext, '')
# HTML
- html = post.get(format='html')
+ html = post.get(content_format='html')
self.assertIn('html', post._content)
self.assertIn('wikitext', post._content)
self.assertIsInstance(html, str)
self.assertNotEqual(html, '')
# Caching (hit)
post._content['html'] = 'something'
- html = post.get(format='html')
+ html = post.get(content_format='html')
self.assertIsInstance(html, str)
self.assertEqual(html, 'something')
self.assertIn('html', post._content)
# Caching (reload)
post._content['html'] = 'something'
- html = post.get(format='html', force=True)
+ html = post.get(content_format='html', force=True)
self.assertIsInstance(html, str)
self.assertNotEqual(html, 'something')
self.assertIn('html', post._content)
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631766
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ibf9e6483378469d410d56050a1843552ccec6357
Gerrit-Change-Number: 631766
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631282 )
Change subject: [bugfix] Avoid duplicates in intersect_generators()
......................................................................
[bugfix] Avoid duplicates in intersect_generators()
Avoid duplicates in intersect_generators().
Add also the possibility to have duplicates if all generators contain
the same page several times.
The solution uses a cache, so for infinite streams there will be memory
issues.
Bug: T263947
Change-Id: I86adcacded455ecd87a35b140dd371cc6caca5a4
---
M pywikibot/tools/__init__.py
M tests/thread_tests.py
2 files changed, 47 insertions(+), 15 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py
index 10bf6bc..b44f246 100644
--- a/pywikibot/tools/__init__.py
+++ b/pywikibot/tools/__init__.py
@@ -692,7 +692,7 @@
.format(thd, thd.queue.qsize()), self._logger)
-def intersect_generators(genlist):
+def intersect_generators(genlist, allow_duplicates=False):
"""
Intersect generators listed in genlist.
@@ -707,6 +707,8 @@
@param genlist: list of page generators
@type genlist: list
+ @param allow_duplicates: allow duplicates if present in all generators
+ @type allow_duplicates: bool
"""
# If any generator is empty, no pages are going to be returned
for source in genlist:
@@ -717,7 +719,8 @@
# Item is cached to check that it is found n_gen
# times before being yielded.
- cache = collections.defaultdict(set)
+ from collections import Counter
+ cache = collections.defaultdict(Counter)
n_gen = len(genlist)
# Class to keep track of alive threads.
@@ -729,6 +732,9 @@
threaded_gen.daemon = True
thrlist.append(threaded_gen)
+ ones = Counter(thrlist)
+ seen = {}
+
while True:
# Get items from queues in a round-robin way.
for t in thrlist:
@@ -736,14 +742,23 @@
# TODO: evaluate if True and timeout is necessary.
item = t.queue.get(True, 0.1)
- # Cache entry is a set of thread.
- # Duplicates from same thread are not counted twice.
- cache[item].add(t)
+ if not allow_duplicates and hash(item) in seen:
+ continue
+
+ # Cache entry is a Counter of ThreadedGenerator objects.
+ cache[item].update([t])
if len(cache[item]) == n_gen:
- yield item
- # Remove item from cache.
- # No chance of seeing it again (see later: early stop).
- cache.pop(item)
+ if allow_duplicates:
+ yield item
+ # Remove item from cache if possible.
+ if all(el == 1 for el in cache[item].values()):
+ cache.pop(item)
+ else:
+ cache[item] -= ones
+ else:
+ yield item
+ cache.pop(item)
+ seen[hash(item)] = True
active = thrlist.active_count()
max_cache = n_gen
diff --git a/tests/thread_tests.py b/tests/thread_tests.py
index 615395b..1516b0e 100644
--- a/tests/thread_tests.py
+++ b/tests/thread_tests.py
@@ -5,6 +5,7 @@
#
# Distributed under the terms of the MIT license.
#
+from collections import Counter
from contextlib import suppress
from tests.aspects import unittest, TestCase
@@ -53,6 +54,18 @@
self.assertCountEqual(set(result), result)
self.assertCountEqual(result, set_result)
+ def assertEqualItertoolsWithDuplicates(self, gens):
+ """Assert intersect_generators result equals Counter intersection."""
+ # If they are a generator, we need to convert to a list
+ # first otherwise the generator is empty the second time.
+ datasets = [list(gen) for gen in gens]
+ counter_result = Counter(datasets[0])
+ for dataset in datasets[1:]:
+ counter_result = counter_result & Counter(dataset)
+ counter_result = list(counter_result.elements())
+ result = list(intersect_generators(datasets, allow_duplicates=True))
+ self.assertCountEqual(counter_result, result)
+
class BasicGeneratorIntersectTestCase(GeneratorIntersectTestCase):
@@ -61,17 +74,21 @@
net = False
def test_intersect_basic(self):
- """Test basic interset without duplicates."""
+ """Test basic intersect without duplicates."""
self.assertEqualItertools(['abc', 'db', 'ba'])
def test_intersect_with_dups(self):
- """Test basic interset with duplicates."""
+ """Test basic intersect with duplicates."""
self.assertEqualItertools(['aabc', 'dddb', 'baa'])
- @unittest.expectedFailure
- def test_intersect_with_dups_failing(self):
- """Failing basic intersect test with duplicates."""
- self.assertEqualItertools(['abb', 'bb'])
+ def test_intersect_with_accepted_dups(self):
+ """Test intersect with duplicates accepted."""
+ self.assertEqualItertoolsWithDuplicates(['abc', 'db', 'ba'])
+ self.assertEqualItertoolsWithDuplicates(['aabc', 'dddb', 'baa'])
+ self.assertEqualItertoolsWithDuplicates(['abb', 'bb'])
+ self.assertEqualItertoolsWithDuplicates(['bb', 'abb'])
+ self.assertEqualItertoolsWithDuplicates(['abbcd', 'abcba'])
+ self.assertEqualItertoolsWithDuplicates(['abcba', 'abbcd'])
if __name__ == '__main__': # pragma: no cover
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/631282
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I86adcacded455ecd87a35b140dd371cc6caca5a4
Gerrit-Change-Number: 631282
Gerrit-PatchSet: 2
Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged