jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808417 )
Change subject: [doc] do not use sphinx 5.0.0-5.0.2
......................................................................
[doc] do not use sphinx 5.0.0-5.0.2
see: https://github.com/sphinx-doc/sphinx/issues/10594
Change-Id: I20a9cac918a8d9c0ea197ae6e994dbde304674b7
---
M docs/conf.py
M docs/requirements-py3.txt
2 files changed, 2 insertions(+), 2 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/docs/conf.py b/docs/conf.py
index 5d80ec2..b42b661 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -45,7 +45,7 @@
# If your documentation needs a minimal Sphinx version, state it here.
#
-needs_sphinx = '4.1'
+needs_sphinx = '4.5'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
diff --git a/docs/requirements-py3.txt b/docs/requirements-py3.txt
index 37333c5..c213a4a 100644
--- a/docs/requirements-py3.txt
+++ b/docs/requirements-py3.txt
@@ -1,4 +1,4 @@
# This is a PIP requirements file for building Sphinx documentation of pywikibot
# requirements.txt is also needed
-sphinx >= 4.5.0,!=5.0.0,!=5.0.1
\ No newline at end of file
+sphinx >= 4.5.0,!=5.0.0,!=5.0.1,!=5.0.2
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808417
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I20a9cac918a8d9c0ea197ae6e994dbde304674b7
Gerrit-Change-Number: 808417
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/807579 )
Change subject: [dist] Provide Built Distribution with Pywikibot (PEP 427)
......................................................................
[dist] Provide Built Distribution with Pywikibot (PEP 427)
- update pip, setuptools, wheel and twine package first
when running make_dist.py wth -upgrade option
- create source distribution and built distibution;
both are stored in the dist folder
- pip prefers built distibution over source distribution automatically;
source distribution is for fallback or older pip versions
Change-Id: I0974ddb736223d6a1dd95054f314c7789c7761a7
---
M make_dist.py
1 file changed, 16 insertions(+), 5 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/make_dist.py b/make_dist.py
index dd67875..21271c3 100644
--- a/make_dist.py
+++ b/make_dist.py
@@ -12,14 +12,21 @@
-remote Upload the package to pypi. This cannot be done if the
Pywikibot version is a development release.
+-upgrade Upgrade distribution packages pip, setuptools, wheel and twine
+ first
+
Usage::
[pwb] make_dist [options]
-.. note:: Requires Python 3.6+
+.. note:: Requires Python 3.6+.
.. versionadded:: 7.3
.. versionchanged:: 7.4
- installs pre-releases over stable versions
+
+ - updates pip, setuptools, wheel and twine packages first
+ - installs pre-releases over stable versions
+ - also creates built distribution together with source distribution
+ - `-upgrade` option was added
"""
#
# (C) Pywikibot team, 2022
@@ -91,20 +98,24 @@
local = '-local' in sys.argv
remote = '-remote' in sys.argv
+ upgrade = '-upgrade' in sys.argv
if remote and 'dev' in __version__:
warning('Distribution must not be a developmental release to upload.')
remote = False
- sys.argv = [sys.argv[0], 'sdist']
- return local, remote
+ sys.argv = [sys.argv[0], 'sdist', 'bdist_wheel']
+ return local, remote, upgrade
def main() -> None:
"""Script entry point."""
- local, remote = handle_args()
+ local, remote, upgrade = handle_args()
copy_files()
+ if upgrade:
+ subprocess.run('python -m pip install --upgrade pip')
+ subprocess.run('pip install --upgrade setuptools wheel twine ')
try:
setup.main() # create a new package
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/807579
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I0974ddb736223d6a1dd95054f314c7789c7761a7
Gerrit-Change-Number: 807579
Gerrit-PatchSet: 3
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/806493 )
Change subject: [IMPR] Improvements for interwiki.py
......................................................................
[IMPR] Improvements for interwiki.py
- add some type hints
- use dict comprehensions to create dicts
- remove While loop in Subject.assemble() because input_choice only
continues if the given input is valid. Therefore 'r'ecject answer
may not be handled because it just continues the for loop.
- simplify sanity check and use error() to print the message
- no OSError is expected in Subject.reportBacklinks(); remove try statement
- simplify InterwikiBot.firstSubject()
- remove InterwikiBot.__len__ and use self.subjects directly in
InterwikiBot.isDone()
Change-Id: I583b002e26db9a6701ee00a53482a4f725e21ee8
---
M scripts/interwiki.py
1 file changed, 81 insertions(+), 108 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/interwiki.py b/scripts/interwiki.py
index 76e75ad..c353b17 100755
--- a/scripts/interwiki.py
+++ b/scripts/interwiki.py
@@ -339,6 +339,7 @@
from collections import Counter, defaultdict
from contextlib import suppress
from textwrap import fill
+from typing import Optional
import pywikibot
from pywikibot import (
@@ -349,6 +350,7 @@
textlib,
titletranslate,
)
+from pywikibot.backports import Iterable
from pywikibot.bot import ListOption, OptionHandler, StandardOption
from pywikibot.cosmetic_changes import moved_links
from pywikibot.exceptions import (
@@ -457,17 +459,16 @@
summary = ''
repository = False
- def note(self, text) -> None:
+ def note(self, text: str) -> None:
"""Output a notification message with.
The text will be printed only if conf.quiet isn't set.
:param text: text to be shown
- :type text: str
"""
if not self.quiet:
pywikibot.output('NOTE: ' + text)
- def readOptions(self, option) -> bool:
+ def readOptions(self, option: str) -> bool:
"""Read all commandline parameters for the global container."""
arg, _, value = option.partition(':')
if not arg.startswith('-'):
@@ -1337,7 +1338,6 @@
errorCount += 1
# See if new{} contains any problematic values
- result = {}
for site, pages in new.items():
if len(pages) > 1:
errorCount += 1
@@ -1345,9 +1345,7 @@
if not errorCount and not self.conf.select:
# no errors, so all lists have only one item
- for site, pages in new.items():
- result[site] = pages[0]
- return result
+ return {site: pages[0] for site, pages in new.items()}
# There are any errors.
if config.interwiki_graph:
@@ -1359,6 +1357,7 @@
if self.conf.autonomous:
return None
+ result = {}
# First loop over the ones that have more solutions
for site, pages in new.items():
if len(pages) <= 1:
@@ -1394,28 +1393,24 @@
page2 = pages[0]
pywikibot.output('Found link to {} in:'.format(page2))
self.whereReport(page2, indent=4)
- while True:
- if acceptall:
- answer = 'a'
- else:
- # TODO: allow answer to repeat previous or go back
- # after a mistake
- answer = pywikibot.input_choice(
- 'What should be done?',
- [('accept', 'a'), ('reject', 'r'),
- ('give up', 'g'), ('accept all', 'l')], 'a',
- automatic_quit=False)
- if answer == 'l': # accept all
- acceptall = True
- answer = 'a'
- if answer == 'a': # accept this one
- result[site] = pages[0]
- break
- if answer == 'g': # give up
- return None
- if answer == 'r': # reject
- # None acceptable
- break
+
+ # TODO: allow answer to repeat previous or go back
+ # after a mistake
+ answer = 'a' if acceptall else pywikibot.input_choice(
+ 'What should be done?',
+ [('accept', 'a'), ('reject', 'r'), ('give up', 'g'),
+ ('accept all', 'l')], 'a', automatic_quit=False)
+
+ if answer == 'l': # accept all
+ acceptall = True
+ answer = 'a'
+
+ if answer == 'a': # accept this one
+ result[site] = pages[0]
+ elif answer == 'g': # give up
+ return None
+ # else reject if None acceptable
+
return result
def finish(self):
@@ -1593,12 +1588,8 @@
# site.
pltmp = new[page.site]
if pltmp != page:
- s = 'None'
- if pltmp is not None:
- s = pltmp
- pywikibot.output(
- 'BUG>>> {} is not in the list of new links! Found {}.'
- .format(page, s))
+ pywikibot.error('{} is not in the list of new links! Found {}.'
+ .format(page, pltmp))
raise SaveError('BUG: sanity check failed')
# Avoid adding an iw link back to itself
@@ -1611,9 +1602,7 @@
del new[stmp]
# Put interwiki links into a map
- old = {}
- for page2 in interwikis:
- old[page2.site] = page2
+ old = {p.site: p for p in interwikis}
# Check what needs to get done
mods, mcomment, adding, removing, modifying = compareLanguages(
@@ -1642,9 +1631,9 @@
self.conf.note('No changes needed on page {}'.format(page))
return False
- pywikibot.output('<<lightpurple>>Updating links on page {}.<<default>>'
- .format(page))
- pywikibot.output('Changes to be made: {}'.format(mods))
+ pywikibot.info('<<lightpurple>>Updating links on page {}.'
+ .format(page))
+ pywikibot.info('Changes to be made: {}'.format(mods))
oldtext = page.get()
template = (page.namespace() == 10)
newtext = textlib.replaceLanguageLinks(oldtext, new,
@@ -1653,14 +1642,12 @@
# This is for now. Later there should be different funktions for each
# kind
if not botMayEdit(page):
+ pywikibot.info('SKIPPING: {} '.format(page), newline=False)
if template:
- pywikibot.output(
- 'SKIPPING: {} should have interwiki links on subpage.'
- .format(page))
+ msg = 'should have interwiki links on subpage.'
else:
- pywikibot.output(
- 'SKIPPING: {} is under construction or to be deleted.'
- .format(page))
+ msg = 'is under construction or to be deleted.'
+ pywikibot.info(msg)
return False
if newtext == oldtext:
@@ -1773,54 +1760,49 @@
# use sets because searching an element is faster than in lists
expectedPages = set(new.values())
expectedSites = set(new)
- try:
- for site in expectedSites - set(updatedSites):
- page = new[site]
- if page.section():
+
+ for site in expectedSites - set(updatedSites):
+ page = new[site]
+ if page.section():
+ continue
+
+ try:
+ linkedPages = {pywikibot.Page(link)
+ for link in page.iterlanglinks()}
+ except NoPageError:
+ pywikibot.warning('Page {} does no longer exist?!'
+ .format(page))
+ break
+
+ # To speed things up, create a dictionary which maps sites
+ # to pages. This assumes that there is only one interwiki
+ # link per language.
+ linkedPagesDict = {p.site: p for p in linkedPages}
+ for expectedPage in expectedPages - linkedPages:
+ if expectedPage == page:
continue
-
try:
- linkedPages = {pywikibot.Page(link)
- for link in page.iterlanglinks()}
- except NoPageError:
+ linkedPage = linkedPagesDict[expectedPage.site]
pywikibot.warning(
- 'Page {} does no longer exist?!'.format(page))
- break
-
- # To speed things up, create a dictionary which maps sites
- # to pages. This assumes that there is only one interwiki
- # link per language.
- linkedPagesDict = {}
- for linkedPage in linkedPages:
- linkedPagesDict[linkedPage.site] = linkedPage
- for expectedPage in expectedPages - linkedPages:
- if expectedPage == page:
- continue
- try:
- linkedPage = linkedPagesDict[expectedPage.site]
- pywikibot.warning(
- '{}: {} does not link to {} but to {}'
- .format(page.site.family.name,
- page, expectedPage, linkedPage))
- except KeyError:
- if not expectedPage.site.is_data_repository():
- pywikibot.warning('{}: {} does not link to {}'
- .format(page.site.family.name,
- page, expectedPage))
- # Check for superfluous links
- for linkedPage in linkedPages:
- if linkedPage in expectedPages:
- continue
- # Check whether there is an alternative page on
- # that language.
- # In this case, it was already reported above.
- if linkedPage.site not in expectedSites:
- pywikibot.warning('{}: {} links to incorrect {}'
+ '{}: {} does not link to {} but to {}'
+ .format(page.site.family.name,
+ page, expectedPage, linkedPage))
+ except KeyError:
+ if not expectedPage.site.is_data_repository():
+ pywikibot.warning('{}: {} does not link to {}'
.format(page.site.family.name,
- page, linkedPage))
-
- except OSError:
- pywikibot.error('could not report backlinks')
+ page, expectedPage))
+ # Check for superfluous links
+ for linkedPage in linkedPages:
+ if linkedPage in expectedPages:
+ continue
+ # Check whether there is an alternative page on
+ # that language.
+ # In this case, it was already reported above.
+ if linkedPage.site not in expectedSites:
+ pywikibot.warning('{}: {} links to incorrect {}'
+ .format(page.site.family.name,
+ page, linkedPage))
class InterwikiBot:
@@ -1935,11 +1917,9 @@
# for loop was exited by break statement
self.pageGenerator = None
- def firstSubject(self):
+ def firstSubject(self) -> Optional[Subject]:
"""Return the first subject that is still being worked on."""
- if self.subjects:
- return self.subjects[0]
- return None
+ return self.subjects[0] if self.subjects else None
def maxOpenSite(self):
"""
@@ -1982,10 +1962,9 @@
self.generateMore(self.conf.maxquerysize - mycount)
except ServerError:
# Could not extract allpages special page?
- pywikibot.output(
- 'ERROR: could not retrieve more pages. '
- 'Will try again in {} seconds'
- .format(timeout))
+ pywikibot.error('could not retrieve more pages. '
+ 'Will try again in {} seconds'
+ .format(timeout))
pywikibot.sleep(timeout)
timeout *= 2
else:
@@ -2049,9 +2028,9 @@
subj.finish()
del self.subjects[i]
- def isDone(self):
+ def isDone(self) -> bool:
"""Check whether there is still more work to do."""
- return not self and self.pageGenerator is None
+ return not self.subjects and self.pageGenerator is None
def plus(self, site, count: int = 1) -> None:
"""Helper routine that the Subject class expects in a counter."""
@@ -2067,10 +2046,6 @@
while not self.isDone():
self.queryStep()
- def __len__(self) -> int:
- """Return length of subjects."""
- return len(self.subjects)
-
def compareLanguages(old, new, insite, summary):
"""Compare changes and setup i18n message."""
@@ -2090,7 +2065,6 @@
return str(d[site])
else:
# Use short format, just the language code
-
def fmt(d, site):
return site.code
@@ -2262,11 +2236,10 @@
namespace=self.next_namespace,
filterredir=False)
- def write_dump(self, iterable, append: bool = True) -> None:
+ def write_dump(self, iterable: Iterable, append: bool = True) -> None:
"""Write dump file.
:param iterable: an iterable of page titles to be dumped.
- :type iterable: iterable
:param append: if a dump already exits, append the page titles to it
if True else overwrite it.
"""
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/806493
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I583b002e26db9a6701ee00a53482a4f725e21ee8
Gerrit-Change-Number: 806493
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808415 )
Change subject: [test] Decrease number of tests in test_two_continuations_limited
......................................................................
[test] Decrease number of tests in test_two_continuations_limited
- Reduce number of tests in
TestPropertyGenerator.test_two_continuations_limited; this decreases
test time by 25%
- use tuple for links and a generator for titles
- test that processed pagedata is less or equal to 'total' links
- test that processed pagedata is more than half of 'total' links
Bug: T311348
Change-Id: Ibeed1a090522f78ce85a71ef61c8372e7bfe9391
---
M tests/api_tests.py
1 file changed, 10 insertions(+), 4 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/tests/api_tests.py b/tests/api_tests.py
index 0a1390c..be3a6e4 100755
--- a/tests/api_tests.py
+++ b/tests/api_tests.py
@@ -609,14 +609,17 @@
def test_two_continuations_limited(self):
"""Test PropertyGenerator with many limited props and continuations."""
+ total = 20
+ increment = total // 4
mainpage = self.get_mainpage()
- links = list(self.site.pagelinks(mainpage, total=30))
- titles = [link.title(with_section=False) for link in links]
+ links = tuple(self.site.pagelinks(mainpage, total=total))
+ titles = (link.title(with_section=False) for link in links)
gen = api.PropertyGenerator(
- site=self.site, prop='info|categoryinfo|langlinks|templates',
+ site=self.site,
+ prop='info|categoryinfo|langlinks|templates',
parameters={'titles': '|'.join(titles)})
# Force the generator into continuation mode
- gen.set_query_increment(5)
+ gen.set_query_increment(increment)
count = 0
for count, pagedata in enumerate(gen, start=1):
@@ -626,6 +629,9 @@
else:
self.assertIn('pageid', pagedata)
self.assertLength(links, count)
+ self.assertGreaterEqual(total, count)
+ # ensure we have enough continuations
+ self.assertGreater(count, total // 2 + 1)
class TestDryQueryGeneratorNamespaceParam(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808415
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ibeed1a090522f78ce85a71ef61c8372e7bfe9391
Gerrit-Change-Number: 808415
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808426 )
Change subject: Revert "[bugfix] Respect limit argument with Board.topics()"
......................................................................
Revert "[bugfix] Respect limit argument with Board.topics()"
This reverts commit 4342aeb9e29e4f365d88b6c1a63fca67d1928cdb.
Reason for revert: too many failing tests since this commit
Change-Id: I95714030f7995502c3eaebfbca1d32e90aa894ac
---
M pywikibot/flow.py
M pywikibot/site/_extensions.py
M tests/flow_tests.py
3 files changed, 39 insertions(+), 57 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 8adbb95..71cc206 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -6,26 +6,29 @@
#
import abc
import datetime
-from typing import Any, Type, Optional, Union
+import logging
+from typing import Any, Type, Union
from urllib.parse import parse_qs, urlparse
import pywikibot
from pywikibot.backports import Dict, Iterator, List, Mapping
-from pywikibot import config
from pywikibot.exceptions import (
LockedPageError,
NoPageError,
UnknownExtensionError,
)
from pywikibot.page import BasePage, PageSourceType, User
-from pywikibot.tools import cached, deprecated_args
+from pywikibot.tools import cached
+logger = logging.getLogger('pywiki.wiki.flow')
+
+
+# Flow page-like objects (boards and topics)
class FlowPage(BasePage, abc.ABC):
"""The base page meta class for the Flow extension.
- Defines Flow page-like object for :class:`Board` and :class:`Topic`.
It cannot be instantiated directly.
"""
@@ -106,29 +109,17 @@
new_params[key] = value
return new_params
- @deprecated_args(limit='total') # since 7.4.0
- def topics(self, *,
- content_format: str = 'wikitext',
- total: Optional[int] = None,
+ def topics(self, content_format: str = 'wikitext', limit: int = 100,
sort_by: str = 'newest',
offset: Union[str, datetime.datetime, None] = None,
- offset_uuid: str = '',
- reverse: bool = False,
- include_offset: bool = False,
- toc_only: bool = False
+ offset_uuid: str = '', reverse: bool = False,
+ include_offset: bool = False, toc_only: bool = False
) -> Iterator['Topic']:
"""Load this board's topics.
- .. versionchanged:: 7.4
- The *total* parameter was added as a per request limit.
- All parameters are keyword only parameters.
- .. deprecated:: 7.4
- The *limit* parameter. Use `-step` global option or
- `config.step` instead.
-
:param content_format: The content format to request the data in;
must be either 'wikitext', 'html', or 'fixed-html'
- :param total: The number of topics to fetch.
+ :param limit: The number of topics to fetch in each request.
:param sort_by: Algorithm to sort topics by;
must be either 'newest' or 'updated'
:param offset: The timestamp to start at (when sortby is 'updated').
@@ -138,25 +129,17 @@
:param toc_only: Whether to only include information for the TOC.
:yield: A generator of this board's topics.
"""
- maxlimit = min(config.step, 100) if config.step > 0 else 100
- request_limit = min(total, maxlimit)
data = self.site.load_topiclist(self, content_format=content_format,
- limit=request_limit, sortby=sort_by,
+ limit=limit, sortby=sort_by,
toconly=toc_only, offset=offset,
offset_id=offset_uuid, reverse=reverse,
include_offset=include_offset)
- count = 0
while data['roots']:
for root in data['roots']:
topic = Topic.from_topiclist_data(self, root, data)
yield topic
-
- count += 1
- if count >= total:
- return
-
- continue_args = self._parse_url(data['links']['pagination'])
- data = self.site.load_topiclist(self, **continue_args)
+ cont_args = self._parse_url(data['links']['pagination'])
+ data = self.site.load_topiclist(self, **cont_args)
def new_topic(self, title: str, content: str,
content_format: str = 'wikitext') -> 'Topic':
diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py
index d855226..220db27 100644
--- a/pywikibot/site/_extensions.py
+++ b/pywikibot/site/_extensions.py
@@ -4,11 +4,10 @@
#
# Distributed under the terms of the MIT license.
#
-from typing import Any, Optional, Union
+from typing import Optional
import pywikibot
from pywikibot.data import api
-from pywikibot.backports import Dict
from pywikibot.echo import Notification
from pywikibot.exceptions import (
APIError,
@@ -372,34 +371,39 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topiclist(self,
- page: 'pywikibot.flow.Board',
- *,
- content_format: str = 'wikitext',
- limit: int = 100,
- sortby: str = 'newest',
- toconly: bool = False,
- offset: Union['pywikibot.Timestamp', str, None] = None,
- offset_id: Optional[str] = None,
- reverse: bool = False,
- include_offset: bool = False) -> Dict[str, Any]:
+ def load_topiclist(
+ self,
+ page,
+ content_format: str = 'wikitext',
+ limit: int = 100,
+ sortby: str = 'newest',
+ toconly: bool = False,
+ offset=None,
+ offset_id=None,
+ reverse: bool = False,
+ include_offset: bool = False
+ ):
"""
Retrieve the topiclist of a Flow board.
- .. versionchanged:: 7.4
- All parameters except *page* are keyword only parameters.
-
:param page: A Flow board
+ :type page: Board
:param content_format: The content format to request the data in.
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each single request.
+ :param limit: The number of topics to fetch in each request.
:param sortby: Algorithm to sort topics by ('newest' or 'updated').
:param toconly: Whether to only include information for the TOC.
+ :type toconly: bool
:param offset: The timestamp to start at (when sortby is 'updated').
+ :type offset: Timestamp or equivalent str
:param offset_id: The topic UUID to start at (when sortby is 'newest').
+ :type offset_id: str (in the form of a UUID)
:param reverse: Whether to reverse the topic ordering.
+ :type reverse: bool
:param include_offset: Whether to include the offset topic.
+ :type include_offset: bool
:return: A dict representing the board's topiclist.
+ :rtype: dict
"""
if offset:
offset = pywikibot.Timestamp.fromtimestampformat(offset)
diff --git a/tests/flow_tests.py b/tests/flow_tests.py
index 3a78317..21575da 100755
--- a/tests/flow_tests.py
+++ b/tests/flow_tests.py
@@ -8,7 +8,6 @@
import unittest
from contextlib import suppress
-from pywikibot import config
from pywikibot.exceptions import NoPageError
from pywikibot.flow import Board, Post, Topic
from tests.aspects import TestCase
@@ -142,14 +141,10 @@
def test_topiclist(self):
"""Test loading of topiclist."""
board = self._page
- total = 7
- for step in (-1, 5, 100):
- with self.subTest(step=step):
- config.step = step
- for i, _ in enumerate(board.topics(total=total), start=1):
- if i > total:
- break
- self.assertEqual(i, total)
+ for i, _ in enumerate(board.topics(limit=7), start=1):
+ if i == 10:
+ break
+ self.assertEqual(i, 10)
class TestFlowFactoryErrors(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808426
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I95714030f7995502c3eaebfbca1d32e90aa894ac
Gerrit-Change-Number: 808426
Gerrit-PatchSet: 1
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828 )
Change subject: [bugfix] Respect limit argument with Board.topics()
......................................................................
[bugfix] Respect limit argument with Board.topics()
- rename 'limit' to 'total' within Board.topics() like in all other
methods and deprecate the old variant
- use keyword arguments only in Board.topics()
- use config.step to determine the request limit. Otherwise use 100 as
the maximum limit per api request
- add a counter to leave the topics method if total is exceeded
- use keyword arguments only in APISite.load_topiclist() except for
FlowPage parameter
- remove type hints in load_topiclist() which are given with the
parameters already
- update test_topiclist test to verify if the Board.topics() only
yields as much items as given by total parameter.
- use various values for config.step.
Bug: T138307
Bug: T138215
Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838
---
M pywikibot/flow.py
M pywikibot/site/_extensions.py
M tests/flow_tests.py
3 files changed, 57 insertions(+), 39 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 71cc206..8adbb95 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -6,29 +6,26 @@
#
import abc
import datetime
-import logging
-from typing import Any, Type, Union
+from typing import Any, Type, Optional, Union
from urllib.parse import parse_qs, urlparse
import pywikibot
from pywikibot.backports import Dict, Iterator, List, Mapping
+from pywikibot import config
from pywikibot.exceptions import (
LockedPageError,
NoPageError,
UnknownExtensionError,
)
from pywikibot.page import BasePage, PageSourceType, User
-from pywikibot.tools import cached
+from pywikibot.tools import cached, deprecated_args
-logger = logging.getLogger('pywiki.wiki.flow')
-
-
-# Flow page-like objects (boards and topics)
class FlowPage(BasePage, abc.ABC):
"""The base page meta class for the Flow extension.
+ Defines Flow page-like object for :class:`Board` and :class:`Topic`.
It cannot be instantiated directly.
"""
@@ -109,17 +106,29 @@
new_params[key] = value
return new_params
- def topics(self, content_format: str = 'wikitext', limit: int = 100,
+ @deprecated_args(limit='total') # since 7.4.0
+ def topics(self, *,
+ content_format: str = 'wikitext',
+ total: Optional[int] = None,
sort_by: str = 'newest',
offset: Union[str, datetime.datetime, None] = None,
- offset_uuid: str = '', reverse: bool = False,
- include_offset: bool = False, toc_only: bool = False
+ offset_uuid: str = '',
+ reverse: bool = False,
+ include_offset: bool = False,
+ toc_only: bool = False
) -> Iterator['Topic']:
"""Load this board's topics.
+ .. versionchanged:: 7.4
+ The *total* parameter was added as a per request limit.
+ All parameters are keyword only parameters.
+ .. deprecated:: 7.4
+ The *limit* parameter. Use `-step` global option or
+ `config.step` instead.
+
:param content_format: The content format to request the data in;
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param total: The number of topics to fetch.
:param sort_by: Algorithm to sort topics by;
must be either 'newest' or 'updated'
:param offset: The timestamp to start at (when sortby is 'updated').
@@ -129,17 +138,25 @@
:param toc_only: Whether to only include information for the TOC.
:yield: A generator of this board's topics.
"""
+ maxlimit = min(config.step, 100) if config.step > 0 else 100
+ request_limit = min(total, maxlimit)
data = self.site.load_topiclist(self, content_format=content_format,
- limit=limit, sortby=sort_by,
+ limit=request_limit, sortby=sort_by,
toconly=toc_only, offset=offset,
offset_id=offset_uuid, reverse=reverse,
include_offset=include_offset)
+ count = 0
while data['roots']:
for root in data['roots']:
topic = Topic.from_topiclist_data(self, root, data)
yield topic
- cont_args = self._parse_url(data['links']['pagination'])
- data = self.site.load_topiclist(self, **cont_args)
+
+ count += 1
+ if count >= total:
+ return
+
+ continue_args = self._parse_url(data['links']['pagination'])
+ data = self.site.load_topiclist(self, **continue_args)
def new_topic(self, title: str, content: str,
content_format: str = 'wikitext') -> 'Topic':
diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py
index 220db27..d855226 100644
--- a/pywikibot/site/_extensions.py
+++ b/pywikibot/site/_extensions.py
@@ -4,10 +4,11 @@
#
# Distributed under the terms of the MIT license.
#
-from typing import Optional
+from typing import Any, Optional, Union
import pywikibot
from pywikibot.data import api
+from pywikibot.backports import Dict
from pywikibot.echo import Notification
from pywikibot.exceptions import (
APIError,
@@ -371,39 +372,34 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topiclist(
- self,
- page,
- content_format: str = 'wikitext',
- limit: int = 100,
- sortby: str = 'newest',
- toconly: bool = False,
- offset=None,
- offset_id=None,
- reverse: bool = False,
- include_offset: bool = False
- ):
+ def load_topiclist(self,
+ page: 'pywikibot.flow.Board',
+ *,
+ content_format: str = 'wikitext',
+ limit: int = 100,
+ sortby: str = 'newest',
+ toconly: bool = False,
+ offset: Union['pywikibot.Timestamp', str, None] = None,
+ offset_id: Optional[str] = None,
+ reverse: bool = False,
+ include_offset: bool = False) -> Dict[str, Any]:
"""
Retrieve the topiclist of a Flow board.
+ .. versionchanged:: 7.4
+ All parameters except *page* are keyword only parameters.
+
:param page: A Flow board
- :type page: Board
:param content_format: The content format to request the data in.
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param limit: The number of topics to fetch in each single request.
:param sortby: Algorithm to sort topics by ('newest' or 'updated').
:param toconly: Whether to only include information for the TOC.
- :type toconly: bool
:param offset: The timestamp to start at (when sortby is 'updated').
- :type offset: Timestamp or equivalent str
:param offset_id: The topic UUID to start at (when sortby is 'newest').
- :type offset_id: str (in the form of a UUID)
:param reverse: Whether to reverse the topic ordering.
- :type reverse: bool
:param include_offset: Whether to include the offset topic.
- :type include_offset: bool
:return: A dict representing the board's topiclist.
- :rtype: dict
"""
if offset:
offset = pywikibot.Timestamp.fromtimestampformat(offset)
diff --git a/tests/flow_tests.py b/tests/flow_tests.py
index 21575da..3a78317 100755
--- a/tests/flow_tests.py
+++ b/tests/flow_tests.py
@@ -8,6 +8,7 @@
import unittest
from contextlib import suppress
+from pywikibot import config
from pywikibot.exceptions import NoPageError
from pywikibot.flow import Board, Post, Topic
from tests.aspects import TestCase
@@ -141,10 +142,14 @@
def test_topiclist(self):
"""Test loading of topiclist."""
board = self._page
- for i, _ in enumerate(board.topics(limit=7), start=1):
- if i == 10:
- break
- self.assertEqual(i, 10)
+ total = 7
+ for step in (-1, 5, 100):
+ with self.subTest(step=step):
+ config.step = step
+ for i, _ in enumerate(board.topics(total=total), start=1):
+ if i > total:
+ break
+ self.assertEqual(i, total)
class TestFlowFactoryErrors(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838
Gerrit-Change-Number: 805828
Gerrit-PatchSet: 7
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged