jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818135 )
Change subject: [bugfix]: fix partial caching in Category.subcategories()
......................................................................
[bugfix]: fix partial caching in Category.subcategories()
Partial caching in category.subcategories() leads to incomplete data
in subsequent calls.
Changes:
- save api calls for categories with no subcategories, based on
cat.categoryinfo['subcats'] info.
At the expense of an extra api call for the base category, for a
cleaner design.
Subcategories have categoryinfo already cached (no extra api calls
needed).
- consider cache valid only if all subcategories have been fetched.
this is guaranteed only if total=none.
- also invalidate cache if called with content=true, after chache is
created with content=false in previous calls.
Bug: T88217
Change-Id: Ifdd4ae5eb1e78764b8d1aaaa286004bcb7705760
---
M pywikibot/page/_pages.py
M tests/category_tests.py
2 files changed, 70 insertions(+), 6 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page/_pages.py b/pywikibot/page/_pages.py
index 9537cd8..b5ebb4a 100644
--- a/pywikibot/page/_pages.py
+++ b/pywikibot/page/_pages.py
@@ -2390,15 +2390,23 @@
:param content: if True, retrieve the content of the current version
of each category description page (default False)
"""
+
+ def is_cache_valid(cache: dict, content: bool) -> bool:
+ return cache['content'] or not content
+
+ if not self.categoryinfo['subcats']:
+ return
+
if not isinstance(recurse, bool) and recurse:
recurse = recurse - 1
- if not hasattr(self, '_subcats'):
- self._subcats = []
- for member in self.site.categorymembers(
+ if (not hasattr(self, '_subcats')
+ or not is_cache_valid(self._subcats, content)):
+ cache = {'data': [], 'content': content}
+
+ for subcat in self.site.categorymembers(
self, member_type='subcat', total=total, content=content):
- subcat = Category(member)
- self._subcats.append(subcat)
+ cache['data'].append(subcat)
yield subcat
if total is not None:
total -= 1
@@ -2415,8 +2423,11 @@
total -= 1
if total == 0:
return
+ else:
+ # cache is valid only if all subcategories are fetched (T88217)
+ self._subcats = cache
else:
- for subcat in self._subcats:
+ for subcat in self._subcats['data']:
yield subcat
if total is not None:
total -= 1
diff --git a/tests/category_tests.py b/tests/category_tests.py
index 2e947c2..3bd8f1f 100755
--- a/tests/category_tests.py
+++ b/tests/category_tests.py
@@ -103,9 +103,62 @@
self.assertIn(c1, subcategories)
self.assertNotIn(c2, subcategories)
+ cat = pywikibot.Category(site, 'Category:Wikipedians by gender')
subcategories_total = list(cat.subcategories(total=2))
self.assertLength(subcategories_total, 2)
+ def test_subcategories_cache_length(self):
+ """Test the subcategories cache length."""
+ site = self.get_site()
+
+ # test cache is valid only if all members of cat are iterated.
+ cat = pywikibot.Category(site, 'Category:Wikipedians by gender')
+ subcategories = list(cat.subcategories(total=2))
+ self.assertLength(subcategories, 2)
+ self.assertFalse(hasattr(cat, '_subcats'))
+
+ subcategories = list(cat.subcategories())
+ self.assertGreater(len(subcategories), 2)
+ self.assertTrue(hasattr(cat, '_subcats'))
+
+ # new cat, no cached data.
+ cat = pywikibot.Category(site, 'Category:Wikipedians by gender')
+
+ # cache not available yet due to partial iteration.
+ gen = cat.subcategories()
+ _ = next(gen)
+ self.assertFalse(hasattr(cat, '_subcats'))
+
+ # cache available.
+ _ = list(gen)
+ self.assertTrue(hasattr(cat, '_subcats'))
+
+ def test_subcategories_cache_content(self):
+ """Test the subcategories cache content."""
+ site = self.get_site()
+ cat = pywikibot.Category(site, 'Category:Wikipedians by gender')
+
+ subcategories = list(cat.subcategories(content=False))
+ cache_id_1 = id(cat._subcats)
+ cache_len_1 = len(subcategories)
+ subcat = subcategories[0]
+ self.assertFalse(subcat.has_content())
+
+ # toggle content.
+ subcategories = list(cat.subcategories(content=True))
+ cache_len_2 = len(subcategories)
+ cache_id_2 = id(cat._subcats)
+ subcat = subcategories[0]
+ self.assertTrue(subcat.has_content())
+ # cache reloaded.
+ self.assertNotEqual(cache_id_1, cache_id_2)
+ self.assertTrue(cache_len_1, cache_len_2)
+
+ # cache not reloaded.
+ _ = list(cat.subcategories(content=True))
+ cache_id_3 = id(cat._subcats)
+ self.assertEqual(cache_id_2, cache_id_3)
+
def test_subcategories_recurse(self):
"""Test the subcategories method with recurse=True."""
site = self.get_site()
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818135
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ifdd4ae5eb1e78764b8d1aaaa286004bcb7705760
Gerrit-Change-Number: 818135
Gerrit-PatchSet: 6
Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818531 )
Change subject: [sphinx] Require sphinx >= 5.1.1
......................................................................
[sphinx] Require sphinx >= 5.1.1
- Require sphinx >= 5.1.1: Problems with sphinx 5.0 and 5.1.0 are
fixed. Therefore require the new version 5.1.1
- Update conf.py
- use searchbox on top of the sidelinks
- split index page
- change permalink icon
Change-Id: I0ff250e3a14e2902a3b197545643e93c633bc5ef
---
M docs/conf.py
M docs/requirements-py3.txt
2 files changed, 41 insertions(+), 25 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/docs/conf.py b/docs/conf.py
index b42b661..e1f2037 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,8 +27,8 @@
from os.path import abspath, dirname, join
-# Deprecated classes will generate warnings as Sphinx processes them. Ignoring
-# them.
+# Deprecated classes will generate warnings as Sphinx processes them.
+# Ignoring them.
warnings.simplefilter(action='ignore', category=FutureWarning)
@@ -45,7 +45,7 @@
# If your documentation needs a minimal Sphinx version, state it here.
#
-needs_sphinx = '4.5'
+needs_sphinx = '5.1.1'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -53,15 +53,14 @@
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.autosectionlabel',
+ 'sphinx.ext.autosummary',
'sphinx.ext.extlinks',
+ # 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon',
'sphinx.ext.todo',
'sphinx.ext.viewcode',
- 'sphinx.ext.autosummary',
- 'sphinx.ext.napoleon',
]
-# Allow lines like "Example:" to be followed by a code block
-napoleon_use_admonition_for_examples = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -69,19 +68,20 @@
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
-source_suffix = ['.rst', '.md']
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
# The encoding of source files.
#
# source_encoding = 'utf-8-sig'
# The master toctree document.
-master_doc = 'index'
+root_doc = 'index'
# General information about the project.
project = pywikibot.__name__.title()
project_copyright = pywikibot.__copyright__ # alias since Python 3.5
-# author = 'test'
+author = 'Pywikibot Team'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -97,7 +97,7 @@
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
@@ -116,7 +116,7 @@
# The reST default role (used for this markup: `text`) to use for all
# documents.
#
-default_role = 'code'
+default_role = 'py:obj'
# If true, '()' will be appended to :func: etc. cross-reference text.
#
@@ -142,7 +142,7 @@
# keep_warnings = False
# If true, `todo` and `todoList` produce output, else they produce nothing.
-# todo_include_todos = False
+todo_include_todos = True
# -- Options for HTML output ----------------------------------------------
@@ -184,7 +184,7 @@
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ['_static']
+html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -200,7 +200,11 @@
# Custom sidebar templates, maps document names to template names.
#
-# html_sidebars = {}
+html_sidebars = {
+ '**': [
+ 'searchbox.html', 'localtoc.html', 'relations.html', 'sourcelink.html',
+ ]
+}
# Additional templates that should be rendered to pages, maps page names to
# template names.
@@ -217,7 +221,7 @@
# If true, the index is split into individual pages for each letter.
#
-# html_split_index = False
+html_split_index = True
# If true, links to the reST sources are added to the pages.
#
@@ -259,7 +263,7 @@
# html_search_scorer = 'scorer.js'
# Output file base name for HTML help builder.
-htmlhelp_basename = 'Pywikibotdoc'
+htmlhelp_basename = project + release.replace('.', '')
# -- Options for LaTeX output ---------------------------------------------
@@ -285,8 +289,8 @@
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
- ('index', 'Pywikibot.tex', 'Pywikibot Documentation',
- 'Pywikibot team', 'manual'),
+ (root_doc, 'Pywikibot.tex', 'Pywikibot Documentation',
+ author, 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -316,8 +320,8 @@
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- ('index', 'pywikibot', 'Pywikibot Documentation',
- ['Pywikibot team'], 1)
+ (root_doc, project, 'Pywikibot Documentation',
+ [author], 1)
]
manpages_url = 'https://www.mediawiki.org/wiki/Manual:Pywikibot/{path}'
@@ -332,8 +336,8 @@
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- ('index', 'Pywikibot', 'Pywikibot Documentation',
- 'Pywikibot team', 'Pywikibot', 'One line description of project.',
+ (root_doc, project, 'Pywikibot Documentation',
+ author, project, 'One line description of project.',
'Miscellaneous'),
]
@@ -353,13 +357,25 @@
#
# texinfo_no_detailmenu = False
+# If false, do not generate in manual @ref nodes.
+#
+# texinfo_cross_references = False
+
+numfig = True
+
# Other settings
+
autodoc_typehints = 'description'
+# Allow lines like "Example:" to be followed by a code block
+napoleon_use_admonition_for_examples = True
+python_use_unqualified_type_names = True
+
# Pywikibot theme style
-html_static_path = ['_static']
+html_permalinks_icon = '#'
html_style = 'css/pywikibot.css'
+
extlinks = {
# MediaWiki API
'api': ('https://www.mediawiki.org/wiki/API:%s', 'API:%s'),
diff --git a/docs/requirements-py3.txt b/docs/requirements-py3.txt
index 7c9fd27..9da69c9 100644
--- a/docs/requirements-py3.txt
+++ b/docs/requirements-py3.txt
@@ -1,4 +1,4 @@
# This is a PIP requirements file for building Sphinx documentation of pywikibot
# requirements.txt is also needed
-sphinx == 4.5.0
\ No newline at end of file
+sphinx >= 5.1.1
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818531
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I0ff250e3a14e2902a3b197545643e93c633bc5ef
Gerrit-Change-Number: 818531
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818121 )
Change subject: [IMPR] Skip the page if it does not exist
......................................................................
[IMPR] Skip the page if it does not exist
If a page does not exist e.g. given with -page option
show the right message and skip instead of printing
"ERROR: Missing or malformed template in page..."
Change-Id: I941ab52d52faaf6c0373e1fc63be8c5432db4bb8
---
M scripts/archivebot.py
1 file changed, 3 insertions(+), 0 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/scripts/archivebot.py b/scripts/archivebot.py
index 19ab8cf..8723eb9 100755
--- a/scripts/archivebot.py
+++ b/scripts/archivebot.py
@@ -829,6 +829,9 @@
namespaces=ns,
content=True)
for pg in gen:
+ if not pg.exists():
+ pywikibot.info('{} does not exist, skipping...'.format(pg))
+ continue
pywikibot.info('\n\n>>> <<lightpurple>>{}<<default>> <<<'
.format(pg.title()))
# Catching exceptions, so that errors in one page do not bail out
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/818121
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I941ab52d52faaf6c0373e1fc63be8c5432db4bb8
Gerrit-Change-Number: 818121
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/817716 )
Change subject: [bugfix] Fix for DiscussionPage.size()
......................................................................
[bugfix] Fix for DiscussionPage.size()
If maxarchivesize is smaller than length of header no archive can be
found even the page does not exist and is empty therefore. To solve
this, DiscussionPage.size() returns 0 if the page does not exists and
has no threads.
Bug: T313886
Change-Id: Iab076d7f8caede9e3caefeb3218112358cd8bfe4
---
M scripts/archivebot.py
1 file changed, 8 insertions(+), 2 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/archivebot.py b/scripts/archivebot.py
index 19ab8cf..aaad6ed 100755
--- a/scripts/archivebot.py
+++ b/scripts/archivebot.py
@@ -392,13 +392,19 @@
return self.is_full(max_archive_size)
def size(self) -> int:
- """
- Return size of talk page threads.
+ """Return size of talk page threads.
Note that this method counts bytes, rather than codepoints
(characters). This corresponds to MediaWiki's definition
of page size.
+
+ .. versionchanged:: 7.6
+ return 0 if archive page neither exists nor has threads
+ (:phab:`T313886`).
"""
+ if not (self.exists() or self.threads):
+ return 0
+
return len(self.header.encode('utf-8')) + sum(t.size()
for t in self.threads)
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/817716
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Iab076d7f8caede9e3caefeb3218112358cd8bfe4
Gerrit-Change-Number: 817716
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: MarcoAurelio <maurelio(a)toolforge.org>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: PotsdamLamb
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/816853 )
Change subject: [bugfix] Fix content=True in page.template()
......................................................................
[bugfix] Fix content=True in page.template()
Discard cache and reload it if cache was loaded without content and
content is required in subsequent function calls.
Fix also page.itertemplates().
Bug: T313736
Change-Id: Id399b56e208cd1f480ff8256c2919f014c99919c
---
M pywikibot/page/_pages.py
1 file changed, 17 insertions(+), 1 deletion(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page/_pages.py b/pywikibot/page/_pages.py
index d130a89..9537cd8 100644
--- a/pywikibot/page/_pages.py
+++ b/pywikibot/page/_pages.py
@@ -19,6 +19,7 @@
#
# Distributed under the terms of the MIT license.
#
+import itertools
import re
from collections import Counter, defaultdict
from contextlib import suppress
@@ -401,6 +402,14 @@
return self.latest_revision.text
+ def has_content(self) -> bool:
+ """
+ Page has been loaded.
+
+ Not existing pages are considered loaded.
+ """
+ return not self.exists() or self._latest_cached_revision() is not None
+
def _latest_cached_revision(self):
"""Get the latest revision if cached and has text, otherwise None."""
if (hasattr(self, '_revid') and self._revid in self._revisions
@@ -1528,6 +1537,12 @@
:param content: bool
"""
# Data might have been preloaded
+ # Delete cache if content is needed and elements have no content
+ if (hasattr(self, '_templates')
+ and content
+ and not all(t.has_content() for t in self._templates)):
+ del self._templates
+
if not hasattr(self, '_templates'):
self._templates = list(self.itertemplates(content=content))
@@ -1549,7 +1564,8 @@
:param content: bool
"""
if hasattr(self, '_templates'):
- return iter(self._templates)
+ return itertools.islice(self.templates(content=content), total)
+
return self.site.pagetemplates(self, total=total, content=content)
def imagelinks(self, total: Optional[int] = None, content: bool = False):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/816853
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Id399b56e208cd1f480ff8256c2919f014c99919c
Gerrit-Change-Number: 816853
Gerrit-PatchSet: 2
Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged