jenkins-bot submitted this change.
[bugfix] Respect limit argument with Board.topics()
- rename 'limit' to 'total' within Board.topics() like in all other
methods and deprecate the old variant
- use keyword arguments only in Board.topics()
- use config.step to determine the request limit. Otherwise use 100 as
the maximum limit per api request
- add a counter to leave the topics method if total is exceeded
- use keyword arguments only in APISite.load_topiclist() except for
FlowPage parameter
- remove type hints in load_topiclist() which are given with the
parameters already
- update test_topiclist test to verify if the Board.topics() only
yields as much items as given by total parameter.
- use various values for config.step.
Bug: T138307
Bug: T138215
Change-Id: I2abbbed785aa52dc84499f6b14cac249f3df612e
---
M pywikibot/flow.py
M pywikibot/site/_extensions.py
M tests/flow_tests.py
3 files changed, 59 insertions(+), 39 deletions(-)
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 4e2984e..ed55212 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -6,29 +6,26 @@
#
import abc
import datetime
-import logging
-from typing import Any, Type, Union
+from typing import Any, Type, Optional, Union
from urllib.parse import parse_qs, urlparse
import pywikibot
from pywikibot.backports import Dict, Iterator, List, Mapping
+from pywikibot import config
from pywikibot.exceptions import (
LockedPageError,
NoPageError,
UnknownExtensionError,
)
from pywikibot.page import BasePage, PageSourceType, User
-from pywikibot.tools import cached
+from pywikibot.tools import cached, deprecated_args
-logger = logging.getLogger('pywiki.wiki.flow')
-
-
-# Flow page-like objects (boards and topics)
class FlowPage(BasePage, abc.ABC):
"""The base page meta class for the Flow extension.
+ Defines Flow page-like object for :class:`Board` and :class:`Topic`.
It cannot be instantiated directly.
"""
@@ -110,17 +107,29 @@
new_params[key] = value
return new_params
- def topics(self, content_format: str = 'wikitext', limit: int = 100,
+ @deprecated_args(limit='total') # since 8.0.0
+ def topics(self, *,
+ content_format: str = 'wikitext',
+ total: Optional[int] = None,
sort_by: str = 'newest',
offset: Union[str, datetime.datetime, None] = None,
- offset_uuid: str = '', reverse: bool = False,
- include_offset: bool = False, toc_only: bool = False
+ offset_uuid: str = '',
+ reverse: bool = False,
+ include_offset: bool = False,
+ toc_only: bool = False
) -> Iterator['Topic']:
"""Load this board's topics.
+ .. versionchanged:: 8.0
+ The *total* parameter was added as a per request limit.
+ All parameters are keyword only parameters.
+ .. deprecated:: 8.0
+ The *limit* parameter. Use ``-step`` global option or
+ :ref:`config.step<Settings to Avoid Server Overload>` instead.
+
:param content_format: The content format to request the data in;
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param total: The number of topics to fetch.
:param sort_by: Algorithm to sort topics by;
must be either 'newest' or 'updated'
:param offset: The timestamp to start at (when sortby is 'updated').
@@ -130,17 +139,25 @@
:param toc_only: Whether to only include information for the TOC.
:yield: A generator of this board's topics.
"""
+ maxlimit = min(config.step, 100) if config.step > 0 else 100
+ request_limit = min(total, maxlimit)
data = self.site.load_topiclist(self, content_format=content_format,
- limit=limit, sortby=sort_by,
+ limit=request_limit, sortby=sort_by,
toconly=toc_only, offset=offset,
offset_id=offset_uuid, reverse=reverse,
include_offset=include_offset)
+ count = 0
while data['roots']:
for root in data['roots']:
topic = Topic.from_topiclist_data(self, root, data)
yield topic
- cont_args = self._parse_url(data['links']['pagination'])
- data = self.site.load_topiclist(self, **cont_args)
+
+ count += 1
+ if count >= total:
+ return
+
+ continue_args = self._parse_url(data['links']['pagination'])
+ data = self.site.load_topiclist(self, **continue_args)
def new_topic(self, title: str, content: str,
content_format: str = 'wikitext') -> 'Topic':
diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py
index 9ba7bf4..8232a9e 100644
--- a/pywikibot/site/_extensions.py
+++ b/pywikibot/site/_extensions.py
@@ -4,9 +4,10 @@
#
# Distributed under the terms of the MIT license.
#
-from typing import Optional
+from typing import Any, Optional, Union
import pywikibot
+from pywikibot.backports import Dict
from pywikibot.data import api
from pywikibot.echo import Notification
from pywikibot.exceptions import (
@@ -374,39 +375,34 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topiclist(
- self,
- page,
- content_format: str = 'wikitext',
- limit: int = 100,
- sortby: str = 'newest',
- toconly: bool = False,
- offset=None,
- offset_id=None,
- reverse: bool = False,
- include_offset: bool = False
- ):
+ def load_topiclist(self,
+ page: 'pywikibot.flow.Board',
+ *,
+ content_format: str = 'wikitext',
+ limit: int = 100,
+ sortby: str = 'newest',
+ toconly: bool = False,
+ offset: Union['pywikibot.Timestamp', str, None] = None,
+ offset_id: Optional[str] = None,
+ reverse: bool = False,
+ include_offset: bool = False) -> Dict[str, Any]:
"""
Retrieve the topiclist of a Flow board.
+ .. versionchanged:: 8.0
+ All parameters except *page* are keyword only parameters.
+
:param page: A Flow board
- :type page: Board
:param content_format: The content format to request the data in.
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param limit: The number of topics to fetch in each single request.
:param sortby: Algorithm to sort topics by ('newest' or 'updated').
:param toconly: Whether to only include information for the TOC.
- :type toconly: bool
:param offset: The timestamp to start at (when sortby is 'updated').
- :type offset: time.Timestamp or equivalent str
:param offset_id: The topic UUID to start at (when sortby is 'newest').
- :type offset_id: str (in the form of a UUID)
:param reverse: Whether to reverse the topic ordering.
- :type reverse: bool
:param include_offset: Whether to include the offset topic.
- :type include_offset: bool
:return: A dict representing the board's topiclist.
- :rtype: dict
"""
if offset:
offset = pywikibot.Timestamp.fromtimestampformat(offset)
diff --git a/tests/flow_tests.py b/tests/flow_tests.py
index 21575da..32ef3fb 100755
--- a/tests/flow_tests.py
+++ b/tests/flow_tests.py
@@ -8,6 +8,7 @@
import unittest
from contextlib import suppress
+from pywikibot import config
from pywikibot.exceptions import NoPageError
from pywikibot.flow import Board, Post, Topic
from tests.aspects import TestCase
@@ -141,10 +142,16 @@
def test_topiclist(self):
"""Test loading of topiclist."""
board = self._page
- for i, _ in enumerate(board.topics(limit=7), start=1):
- if i == 10:
- break
- self.assertEqual(i, 10)
+ total = 7
+ saved_step = config.step
+ for step in (-1, 5, 100):
+ with self.subTest(step=step):
+ config.step = step
+ for i, _ in enumerate(board.topics(total=total), start=1):
+ if i > total:
+ break
+ self.assertEqual(i, total)
+ config.step = saved_step
class TestFlowFactoryErrors(TestCase):
To view, visit change 808420. To unsubscribe, or for help writing mail filters, visit settings.