jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828 )
Change subject: [bugfix] Respect limit argument with Board.topics()
......................................................................
[bugfix] Respect limit argument with Board.topics()
- rename 'limit' to 'total' within Board.topics() like in all other
methods and deprecate the old variant
- use keyword arguments only in Board.topics()
- use config.step to determine the request limit. Otherwise use 100 as
the maximum limit per api request
- add a counter to leave the topics method if total is exceeded
- use keyword arguments only in APISite.load_topiclist() except for
FlowPage parameter
- remove type hints in load_topiclist() which are given with the
parameters already
- update test_topiclist test to verify if the Board.topics() only
yields as much items as given by total parameter.
- use various values for config.step.
Bug: T138307
Bug: T138215
Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838
---
M pywikibot/flow.py
M pywikibot/site/_extensions.py
M tests/flow_tests.py
3 files changed, 57 insertions(+), 39 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 71cc206..8adbb95 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -6,29 +6,26 @@
#
import abc
import datetime
-import logging
-from typing import Any, Type, Union
+from typing import Any, Type, Optional, Union
from urllib.parse import parse_qs, urlparse
import pywikibot
from pywikibot.backports import Dict, Iterator, List, Mapping
+from pywikibot import config
from pywikibot.exceptions import (
LockedPageError,
NoPageError,
UnknownExtensionError,
)
from pywikibot.page import BasePage, PageSourceType, User
-from pywikibot.tools import cached
+from pywikibot.tools import cached, deprecated_args
-logger = logging.getLogger('pywiki.wiki.flow')
-
-
-# Flow page-like objects (boards and topics)
class FlowPage(BasePage, abc.ABC):
"""The base page meta class for the Flow extension.
+ Defines Flow page-like object for :class:`Board` and :class:`Topic`.
It cannot be instantiated directly.
"""
@@ -109,17 +106,29 @@
new_params[key] = value
return new_params
- def topics(self, content_format: str = 'wikitext', limit: int = 100,
+ @deprecated_args(limit='total') # since 7.4.0
+ def topics(self, *,
+ content_format: str = 'wikitext',
+ total: Optional[int] = None,
sort_by: str = 'newest',
offset: Union[str, datetime.datetime, None] = None,
- offset_uuid: str = '', reverse: bool = False,
- include_offset: bool = False, toc_only: bool = False
+ offset_uuid: str = '',
+ reverse: bool = False,
+ include_offset: bool = False,
+ toc_only: bool = False
) -> Iterator['Topic']:
"""Load this board's topics.
+ .. versionchanged:: 7.4
+ The *total* parameter was added as a per request limit.
+ All parameters are keyword only parameters.
+ .. deprecated:: 7.4
+ The *limit* parameter. Use `-step` global option or
+ `config.step` instead.
+
:param content_format: The content format to request the data in;
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param total: The number of topics to fetch.
:param sort_by: Algorithm to sort topics by;
must be either 'newest' or 'updated'
:param offset: The timestamp to start at (when sortby is 'updated').
@@ -129,17 +138,25 @@
:param toc_only: Whether to only include information for the TOC.
:yield: A generator of this board's topics.
"""
+ maxlimit = min(config.step, 100) if config.step > 0 else 100
+ request_limit = min(total, maxlimit)
data = self.site.load_topiclist(self, content_format=content_format,
- limit=limit, sortby=sort_by,
+ limit=request_limit, sortby=sort_by,
toconly=toc_only, offset=offset,
offset_id=offset_uuid, reverse=reverse,
include_offset=include_offset)
+ count = 0
while data['roots']:
for root in data['roots']:
topic = Topic.from_topiclist_data(self, root, data)
yield topic
- cont_args = self._parse_url(data['links']['pagination'])
- data = self.site.load_topiclist(self, **cont_args)
+
+ count += 1
+ if count >= total:
+ return
+
+ continue_args = self._parse_url(data['links']['pagination'])
+ data = self.site.load_topiclist(self, **continue_args)
def new_topic(self, title: str, content: str,
content_format: str = 'wikitext') -> 'Topic':
diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py
index 220db27..d855226 100644
--- a/pywikibot/site/_extensions.py
+++ b/pywikibot/site/_extensions.py
@@ -4,10 +4,11 @@
#
# Distributed under the terms of the MIT license.
#
-from typing import Optional
+from typing import Any, Optional, Union
import pywikibot
from pywikibot.data import api
+from pywikibot.backports import Dict
from pywikibot.echo import Notification
from pywikibot.exceptions import (
APIError,
@@ -371,39 +372,34 @@
return data['flow']['view-topiclist']['result']['topiclist']
@need_extension('Flow')
- def load_topiclist(
- self,
- page,
- content_format: str = 'wikitext',
- limit: int = 100,
- sortby: str = 'newest',
- toconly: bool = False,
- offset=None,
- offset_id=None,
- reverse: bool = False,
- include_offset: bool = False
- ):
+ def load_topiclist(self,
+ page: 'pywikibot.flow.Board',
+ *,
+ content_format: str = 'wikitext',
+ limit: int = 100,
+ sortby: str = 'newest',
+ toconly: bool = False,
+ offset: Union['pywikibot.Timestamp', str, None] = None,
+ offset_id: Optional[str] = None,
+ reverse: bool = False,
+ include_offset: bool = False) -> Dict[str, Any]:
"""
Retrieve the topiclist of a Flow board.
+ .. versionchanged:: 7.4
+ All parameters except *page* are keyword only parameters.
+
:param page: A Flow board
- :type page: Board
:param content_format: The content format to request the data in.
must be either 'wikitext', 'html', or 'fixed-html'
- :param limit: The number of topics to fetch in each request.
+ :param limit: The number of topics to fetch in each single request.
:param sortby: Algorithm to sort topics by ('newest' or 'updated').
:param toconly: Whether to only include information for the TOC.
- :type toconly: bool
:param offset: The timestamp to start at (when sortby is 'updated').
- :type offset: Timestamp or equivalent str
:param offset_id: The topic UUID to start at (when sortby is 'newest').
- :type offset_id: str (in the form of a UUID)
:param reverse: Whether to reverse the topic ordering.
- :type reverse: bool
:param include_offset: Whether to include the offset topic.
- :type include_offset: bool
:return: A dict representing the board's topiclist.
- :rtype: dict
"""
if offset:
offset = pywikibot.Timestamp.fromtimestampformat(offset)
diff --git a/tests/flow_tests.py b/tests/flow_tests.py
index 21575da..3a78317 100755
--- a/tests/flow_tests.py
+++ b/tests/flow_tests.py
@@ -8,6 +8,7 @@
import unittest
from contextlib import suppress
+from pywikibot import config
from pywikibot.exceptions import NoPageError
from pywikibot.flow import Board, Post, Topic
from tests.aspects import TestCase
@@ -141,10 +142,14 @@
def test_topiclist(self):
"""Test loading of topiclist."""
board = self._page
- for i, _ in enumerate(board.topics(limit=7), start=1):
- if i == 10:
- break
- self.assertEqual(i, 10)
+ total = 7
+ for step in (-1, 5, 100):
+ with self.subTest(step=step):
+ config.step = step
+ for i, _ in enumerate(board.topics(total=total), start=1):
+ if i > total:
+ break
+ self.assertEqual(i, total)
class TestFlowFactoryErrors(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/805828
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I33991c1f43b37681a82499a7c36a9bbe0e13e838
Gerrit-Change-Number: 805828
Gerrit-PatchSet: 7
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Happy5214 <happy5214(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged