jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/756165 )
Change subject: [bugfix] Always use low limit for bulk load of redirects
......................................................................
[bugfix] Always use low limit for bulk load of redirects
- Always load 50 pages at once with _next_redirect_group
- rename apiQ to chunk
- print a dot every 500 pages
- only yield multiple or broken redirects with get_redirects_via_api();
this ensures that no page title is printed if redirect.py does
not process it.
Bug: T299859
Change-Id: I7be7b5bbdc35ec98d7007792d95971202119ffcc
---
M scripts/redirect.py
1 file changed, 18 insertions(+), 10 deletions(-)
Approvals:
Meno25: Looks good to me, but someone else must approve
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/redirect.py b/scripts/redirect.py
index ec74455..a836bf9 100755
--- a/scripts/redirect.py
+++ b/scripts/redirect.py
@@ -213,22 +213,28 @@
def _next_redirect_group(self) -> Generator[List[pywikibot.Page], None,
None]:
- """Generator that yields batches of 500 redirects as a
list."""
- apiQ = []
+ """Generator that yields batches of 50 redirects as a
list."""
+ chunk = []
+ chunks = 0
for page in self.get_redirect_pages_via_api():
- apiQ.append(str(page.pageid))
- if len(apiQ) >= 500:
- pywikibot.output('.', newline=False)
- yield apiQ
- apiQ = []
- if apiQ:
- yield apiQ
+ chunk.append(str(page.pageid))
+ if len(chunk) >= 50: # T299859
+ chunks += 1
+ if not chunks % 10:
+ pywikibot.output('.', newline=False)
+ yield chunk
+ chunk.clear()
+ if chunk:
+ yield chunk
def get_redirects_via_api(self, maxlen=8) -> Generator[Tuple[
str, Optional[int], str, Optional[str]], None, None]:
r"""
Return a generator that yields tuples of data about redirect Pages.
+ .. versionchanged:: 7.0
+ only yield tuple if type of redirect is not 1 (normal redirect)
+
The description of returned tuple items is as follows:
:[0]: page title of a redirect page
@@ -281,7 +287,9 @@
result += 1
final = redirects[final]
- yield (redirect, result, target, final)
+ # only yield multiple or broken redirects
+ if result != 1:
+ yield redirect, result, target, final
def retrieve_broken_redirects(self) -> Generator[
Union[str, pywikibot.Page], None, None]:
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/756165
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I7be7b5bbdc35ec98d7007792d95971202119ffcc
Gerrit-Change-Number: 756165
Gerrit-PatchSet: 3
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: Meno25 <meno25mail(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged