jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/808420 )
Change subject: [bugfix] Respect limit argument with Board.topics() ......................................................................
[bugfix] Respect limit argument with Board.topics()
- rename 'limit' to 'total' within Board.topics() like in all other methods and deprecate the old variant - use keyword arguments only in Board.topics() - use config.step to determine the request limit. Otherwise use 100 as the maximum limit per api request - add a counter to leave the topics method if total is exceeded
- use keyword arguments only in APISite.load_topiclist() except for FlowPage parameter - remove type hints in load_topiclist() which are given with the parameters already
- update test_topiclist test to verify if the Board.topics() only yields as much items as given by total parameter. - use various values for config.step.
Bug: T138307 Bug: T138215 Change-Id: I2abbbed785aa52dc84499f6b14cac249f3df612e --- M pywikibot/flow.py M pywikibot/site/_extensions.py M tests/flow_tests.py 3 files changed, 59 insertions(+), 39 deletions(-)
Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py index 4e2984e..ed55212 100644 --- a/pywikibot/flow.py +++ b/pywikibot/flow.py @@ -6,29 +6,26 @@ # import abc import datetime -import logging -from typing import Any, Type, Union +from typing import Any, Type, Optional, Union from urllib.parse import parse_qs, urlparse
import pywikibot from pywikibot.backports import Dict, Iterator, List, Mapping +from pywikibot import config from pywikibot.exceptions import ( LockedPageError, NoPageError, UnknownExtensionError, ) from pywikibot.page import BasePage, PageSourceType, User -from pywikibot.tools import cached +from pywikibot.tools import cached, deprecated_args
-logger = logging.getLogger('pywiki.wiki.flow') - - -# Flow page-like objects (boards and topics) class FlowPage(BasePage, abc.ABC):
"""The base page meta class for the Flow extension.
+ Defines Flow page-like object for :class:`Board` and :class:`Topic`. It cannot be instantiated directly. """
@@ -110,17 +107,29 @@ new_params[key] = value return new_params
- def topics(self, content_format: str = 'wikitext', limit: int = 100, + @deprecated_args(limit='total') # since 8.0.0 + def topics(self, *, + content_format: str = 'wikitext', + total: Optional[int] = None, sort_by: str = 'newest', offset: Union[str, datetime.datetime, None] = None, - offset_uuid: str = '', reverse: bool = False, - include_offset: bool = False, toc_only: bool = False + offset_uuid: str = '', + reverse: bool = False, + include_offset: bool = False, + toc_only: bool = False ) -> Iterator['Topic']: """Load this board's topics.
+ .. versionchanged:: 8.0 + The *total* parameter was added as a per request limit. + All parameters are keyword only parameters. + .. deprecated:: 8.0 + The *limit* parameter. Use ``-step`` global option or + :ref:`config.step<Settings to Avoid Server Overload>` instead. + :param content_format: The content format to request the data in; must be either 'wikitext', 'html', or 'fixed-html' - :param limit: The number of topics to fetch in each request. + :param total: The number of topics to fetch. :param sort_by: Algorithm to sort topics by; must be either 'newest' or 'updated' :param offset: The timestamp to start at (when sortby is 'updated'). @@ -130,17 +139,25 @@ :param toc_only: Whether to only include information for the TOC. :yield: A generator of this board's topics. """ + maxlimit = min(config.step, 100) if config.step > 0 else 100 + request_limit = min(total, maxlimit) data = self.site.load_topiclist(self, content_format=content_format, - limit=limit, sortby=sort_by, + limit=request_limit, sortby=sort_by, toconly=toc_only, offset=offset, offset_id=offset_uuid, reverse=reverse, include_offset=include_offset) + count = 0 while data['roots']: for root in data['roots']: topic = Topic.from_topiclist_data(self, root, data) yield topic - cont_args = self._parse_url(data['links']['pagination']) - data = self.site.load_topiclist(self, **cont_args) + + count += 1 + if count >= total: + return + + continue_args = self._parse_url(data['links']['pagination']) + data = self.site.load_topiclist(self, **continue_args)
def new_topic(self, title: str, content: str, content_format: str = 'wikitext') -> 'Topic': diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py index 9ba7bf4..8232a9e 100644 --- a/pywikibot/site/_extensions.py +++ b/pywikibot/site/_extensions.py @@ -4,9 +4,10 @@ # # Distributed under the terms of the MIT license. # -from typing import Optional +from typing import Any, Optional, Union
import pywikibot +from pywikibot.backports import Dict from pywikibot.data import api from pywikibot.echo import Notification from pywikibot.exceptions import ( @@ -374,39 +375,34 @@ return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow') - def load_topiclist( - self, - page, - content_format: str = 'wikitext', - limit: int = 100, - sortby: str = 'newest', - toconly: bool = False, - offset=None, - offset_id=None, - reverse: bool = False, - include_offset: bool = False - ): + def load_topiclist(self, + page: 'pywikibot.flow.Board', + *, + content_format: str = 'wikitext', + limit: int = 100, + sortby: str = 'newest', + toconly: bool = False, + offset: Union['pywikibot.Timestamp', str, None] = None, + offset_id: Optional[str] = None, + reverse: bool = False, + include_offset: bool = False) -> Dict[str, Any]: """ Retrieve the topiclist of a Flow board.
+ .. versionchanged:: 8.0 + All parameters except *page* are keyword only parameters. + :param page: A Flow board - :type page: Board :param content_format: The content format to request the data in. must be either 'wikitext', 'html', or 'fixed-html' - :param limit: The number of topics to fetch in each request. + :param limit: The number of topics to fetch in each single request. :param sortby: Algorithm to sort topics by ('newest' or 'updated'). :param toconly: Whether to only include information for the TOC. - :type toconly: bool :param offset: The timestamp to start at (when sortby is 'updated'). - :type offset: time.Timestamp or equivalent str :param offset_id: The topic UUID to start at (when sortby is 'newest'). - :type offset_id: str (in the form of a UUID) :param reverse: Whether to reverse the topic ordering. - :type reverse: bool :param include_offset: Whether to include the offset topic. - :type include_offset: bool :return: A dict representing the board's topiclist. - :rtype: dict """ if offset: offset = pywikibot.Timestamp.fromtimestampformat(offset) diff --git a/tests/flow_tests.py b/tests/flow_tests.py index 21575da..32ef3fb 100755 --- a/tests/flow_tests.py +++ b/tests/flow_tests.py @@ -8,6 +8,7 @@ import unittest from contextlib import suppress
+from pywikibot import config from pywikibot.exceptions import NoPageError from pywikibot.flow import Board, Post, Topic from tests.aspects import TestCase @@ -141,10 +142,16 @@ def test_topiclist(self): """Test loading of topiclist.""" board = self._page - for i, _ in enumerate(board.topics(limit=7), start=1): - if i == 10: - break - self.assertEqual(i, 10) + total = 7 + saved_step = config.step + for step in (-1, 5, 100): + with self.subTest(step=step): + config.step = step + for i, _ in enumerate(board.topics(total=total), start=1): + if i > total: + break + self.assertEqual(i, total) + config.step = saved_step
class TestFlowFactoryErrors(TestCase):