jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828 ) Change subject: [bugfix] Respect limit argument with Board.topics() ...................................................................... [bugfix] Respect limit argument with Board.topics() - rename 'limit' to 'total' within Board.topics() like in all other methods and deprecate the old variant - use keyword arguments only in Board.topics() - use config.step to determine the request limit. Otherwise use 100 as the maximum limit per api request - add a counter to leave the topics method if total is exceeded - use keyword arguments only in APISite.load_topiclist() except for FlowPage parameter - remove type hints in load_topiclist() which are given with the parameters already - update test_topiclist test to verify if the Board.topics() only yields as much items as given by total parameter. - use various values for config.step. Bug: T138307 Bug: T138215 Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838 --- M pywikibot/flow.py M pywikibot/site/_extensions.py M tests/flow_tests.py 3 files changed, 57 insertions(+), 39 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/flow.py b/pywikibot/flow.py index 71cc206..8adbb95 100644 --- a/pywikibot/flow.py +++ b/pywikibot/flow.py @@ -6,29 +6,26 @@ # import abc import datetime -import logging -from typing import Any, Type, Union +from typing import Any, Type, Optional, Union from urllib.parse import parse_qs, urlparse import pywikibot from pywikibot.backports import Dict, Iterator, List, Mapping +from pywikibot import config from pywikibot.exceptions import ( LockedPageError, NoPageError, UnknownExtensionError, ) from pywikibot.page import BasePage, PageSourceType, User -from pywikibot.tools import cached +from pywikibot.tools import cached, deprecated_args -logger = logging.getLogger('pywiki.wiki.flow') - - -# Flow page-like objects (boards and topics) class FlowPage(BasePage, abc.ABC): """The base page meta class for the Flow extension. + Defines Flow page-like object for :class:`Board` and :class:`Topic`. It cannot be instantiated directly. """ @@ -109,17 +106,29 @@ new_params[key] = value return new_params - def topics(self, content_format: str = 'wikitext', limit: int = 100, + @deprecated_args(limit='total') # since 7.4.0 + def topics(self, *, + content_format: str = 'wikitext', + total: Optional[int] = None, sort_by: str = 'newest', offset: Union[str, datetime.datetime, None] = None, - offset_uuid: str = '', reverse: bool = False, - include_offset: bool = False, toc_only: bool = False + offset_uuid: str = '', + reverse: bool = False, + include_offset: bool = False, + toc_only: bool = False ) -> Iterator['Topic']: """Load this board's topics. + .. versionchanged:: 7.4 + The *total* parameter was added as a per request limit. + All parameters are keyword only parameters. + .. deprecated:: 7.4 + The *limit* parameter. Use `-step` global option or + `config.step` instead. + :param content_format: The content format to request the data in; must be either 'wikitext', 'html', or 'fixed-html' - :param limit: The number of topics to fetch in each request. + :param total: The number of topics to fetch. :param sort_by: Algorithm to sort topics by; must be either 'newest' or 'updated' :param offset: The timestamp to start at (when sortby is 'updated'). @@ -129,17 +138,25 @@ :param toc_only: Whether to only include information for the TOC. :yield: A generator of this board's topics. """ + maxlimit = min(config.step, 100) if config.step > 0 else 100 + request_limit = min(total, maxlimit) data = self.site.load_topiclist(self, content_format=content_format, - limit=limit, sortby=sort_by, + limit=request_limit, sortby=sort_by, toconly=toc_only, offset=offset, offset_id=offset_uuid, reverse=reverse, include_offset=include_offset) + count = 0 while data['roots']: for root in data['roots']: topic = Topic.from_topiclist_data(self, root, data) yield topic - cont_args = self._parse_url(data['links']['pagination']) - data = self.site.load_topiclist(self, **cont_args) + + count += 1 + if count >= total: + return + + continue_args = self._parse_url(data['links']['pagination']) + data = self.site.load_topiclist(self, **continue_args) def new_topic(self, title: str, content: str, content_format: str = 'wikitext') -> 'Topic': diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py index 220db27..d855226 100644 --- a/pywikibot/site/_extensions.py +++ b/pywikibot/site/_extensions.py @@ -4,10 +4,11 @@ # # Distributed under the terms of the MIT license. # -from typing import Optional +from typing import Any, Optional, Union import pywikibot from pywikibot.data import api +from pywikibot.backports import Dict from pywikibot.echo import Notification from pywikibot.exceptions import ( APIError, @@ -371,39 +372,34 @@ return data['flow']['view-topiclist']['result']['topiclist'] @need_extension('Flow') - def load_topiclist( - self, - page, - content_format: str = 'wikitext', - limit: int = 100, - sortby: str = 'newest', - toconly: bool = False, - offset=None, - offset_id=None, - reverse: bool = False, - include_offset: bool = False - ): + def load_topiclist(self, + page: 'pywikibot.flow.Board', + *, + content_format: str = 'wikitext', + limit: int = 100, + sortby: str = 'newest', + toconly: bool = False, + offset: Union['pywikibot.Timestamp', str, None] = None, + offset_id: Optional[str] = None, + reverse: bool = False, + include_offset: bool = False) -> Dict[str, Any]: """ Retrieve the topiclist of a Flow board. + .. versionchanged:: 7.4 + All parameters except *page* are keyword only parameters. + :param page: A Flow board - :type page: Board :param content_format: The content format to request the data in. must be either 'wikitext', 'html', or 'fixed-html' - :param limit: The number of topics to fetch in each request. + :param limit: The number of topics to fetch in each single request. :param sortby: Algorithm to sort topics by ('newest' or 'updated'). :param toconly: Whether to only include information for the TOC. - :type toconly: bool :param offset: The timestamp to start at (when sortby is 'updated'). - :type offset: Timestamp or equivalent str :param offset_id: The topic UUID to start at (when sortby is 'newest'). - :type offset_id: str (in the form of a UUID) :param reverse: Whether to reverse the topic ordering. - :type reverse: bool :param include_offset: Whether to include the offset topic. - :type include_offset: bool :return: A dict representing the board's topiclist. - :rtype: dict """ if offset: offset = pywikibot.Timestamp.fromtimestampformat(offset) diff --git a/tests/flow_tests.py b/tests/flow_tests.py index 21575da..3a78317 100755 --- a/tests/flow_tests.py +++ b/tests/flow_tests.py @@ -8,6 +8,7 @@ import unittest from contextlib import suppress +from pywikibot import config from pywikibot.exceptions import NoPageError from pywikibot.flow import Board, Post, Topic from tests.aspects import TestCase @@ -141,10 +142,14 @@ def test_topiclist(self): """Test loading of topiclist.""" board = self._page - for i, _ in enumerate(board.topics(limit=7), start=1): - if i == 10: - break - self.assertEqual(i, 10) + total = 7 + for step in (-1, 5, 100): + with self.subTest(step=step): + config.step = step + for i, _ in enumerate(board.topics(total=total), start=1): + if i > total: + break + self.assertEqual(i, total) class TestFlowFactoryErrors(TestCase): -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838 Gerrit-Change-Number: 805828 Gerrit-PatchSet: 7 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged