jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/432773 )
Change subject: [IMPR] Simplify creating dictionaries
......................................................................
[IMPR] Simplify creating dictionaries
Change-Id: If3f88247b2b62ca70a5d5ac02025d0c0c74c71a4
---
M scripts/casechecker.py
1 file changed, 6 insertions(+), 8 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/casechecker.py b/scripts/casechecker.py
index 92aa7a6..bf2b817 100755
--- a/scripts/casechecker.py
+++ b/scripts/casechecker.py
@@ -171,20 +171,18 @@
self.titleList = [self.Page(t) for t in f]
self.failedTitles += '.failed'
+ ziplist = zip(self.localSuspects, self.latinSuspects)
self.lclToLatDict = {
- ord(self.localSuspects[i]): self.latinSuspects[i]
- for i in xrange(len(self.localSuspects))}
+ ord(local): latin for local, latin in ziplist}
self.latToLclDict = {
- ord(self.latinSuspects[i]): self.localSuspects[i]
- for i in xrange(len(self.localSuspects))}
+ ord(latin): local for local, latin in ziplist}
if self.localKeyboard is not None:
+ ziplist = zip(self.localKeyboard, self.latinKeyboard)
self.lclToLatKeybDict = {
- ord(self.localKeyboard[i]): self.latinKeyboard[i]
- for i in xrange(len(self.localKeyboard))}
+ ord(local): latin for local, latin in ziplist}
self.latToLclKeybDict = {
- ord(self.latinKeyboard[i]): self.localKeyboard[i]
- for i in xrange(len(self.localKeyboard))}
+ ord(latin): local for local, latin in ziplist}
else:
self.lclToLatKeybDict = {}
self.latToLclKeybDict = {}
--
To view, visit https://gerrit.wikimedia.org/r/432773
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: If3f88247b2b62ca70a5d5ac02025d0c0c74c71a4
Gerrit-Change-Number: 432773
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/435676 )
Change subject: Sphinx autodoc workaround for Family classproperty-s
......................................................................
Sphinx autodoc workaround for Family classproperty-s
Sphinx uses getattr() on the class in order to get the attribute.
This works for normal instance properties, since it gets the property
instance itself, but for classproperties, it gets whatever the
classproperty code returns. This not only loses all documentation
the classproperty code have, and can potentially cause side effects.
Other methods to workaround the issue are described in:
https://stackoverflow.com/q/22357961. This includes subclassing the
return value (unknown consequences to user), metaclass property
(not extensible), proxy object (error-prone and ugly),and indirect
get method (breaks backwards-compatibility entirely). This workaround
makes Sphinx uses a custom getattr we defined in docs/conf.py for
getting attributes of Family subclass, where it iterates through the
MRO looking for a definition. If it's defined as a classproperty,
it returns the classproperty instance instead; else it falls back
to sphinx's safe_getattr().
Other changes in this patch include:
* make all families defined in family.py abstract and
un-instantiatable.
* make classproperty fetch its docstring from its wrapped function
* make docs/conf.py have its `from script.` import after insert to
sys.path so that it is easier to run sphinx locally.
* make SubdomainFamily.codes no longer treat empty languages_by_size
as valid, because empty languages_by_size is always defined via
MRO and `hasattr(cls, 'languages_by_size')` is never False. This
also prevents SubdomainFamily.codes being accessed directly.
Change-Id: I790580b8d5d9e6349a26ab52c8859a7148441bff
---
M docs/conf.py
M pywikibot/family.py
M pywikibot/tools/__init__.py
3 files changed, 36 insertions(+), 10 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/docs/conf.py b/docs/conf.py
index 14b44e8..dadc26c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -22,9 +22,6 @@
import re
import sys
-from scripts.cosmetic_changes import warning
-
-
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -262,9 +259,10 @@
os.environ['PYWIKIBOT2_NO_USER_CONFIG'] = '1'
-def pywikibot_script_docstring_fixups(
- app, what, name, obj, options, lines):
+def pywikibot_script_docstring_fixups(app, what, name, obj, options, lines):
"""Pywikibot specific conversions."""
+ from scripts.cosmetic_changes import warning
+
if what != "module":
return
@@ -326,10 +324,35 @@
return skip or name in exclusions
+def pywikibot_family_classproperty_getattr(obj, name, *defargs):
+ """Custom getattr() to get classproperty instances."""
+ from sphinx.util.inspect import safe_getattr
+
+ from pywikibot.family import Family
+ from pywikibot.tools import classproperty
+
+ if not isinstance(obj, type) or not issubclass(obj, Family):
+ return safe_getattr(obj, name, *defargs)
+
+ for base_class in obj.__mro__:
+ try:
+ prop = base_class.__dict__[name]
+ except KeyError:
+ continue
+
+ if not isinstance(prop, classproperty):
+ return safe_getattr(obj, name, *defargs)
+
+ return prop
+ else:
+ return safe_getattr(obj, name, *defargs)
+
+
def setup(app):
"""Implicit Sphinx extension hook."""
app.connect('autodoc-process-docstring', pywikibot_script_docstring_fixups)
app.connect('autodoc-skip-member', pywikibot_skip_members)
+ app.add_autodoc_attrgetter(type, pywikibot_family_classproperty_getattr)
pywikibot_env()
diff --git a/pywikibot/family.py b/pywikibot/family.py
index e5513c0..ae52a7b 100644
--- a/pywikibot/family.py
+++ b/pywikibot/family.py
@@ -46,9 +46,11 @@
def __new__(cls):
"""Allocator."""
- if cls is Family:
- raise TypeError('Base Family class cannot be instantiated; '
- 'subclass it instead')
+ # any Family class defined in this file are abstract
+ if cls in globals().values():
+ raise TypeError(
+ 'Abstract Family class {0} cannot be instantiated; '
+ 'subclass it instead'.format(cls.__name__))
# Override classproperty
cls.instance = super(Family, cls).__new__(cls)
@@ -1544,7 +1546,7 @@
@classproperty
def codes(cls):
"""Property listing family codes."""
- if hasattr(cls, 'languages_by_size'):
+ if cls.languages_by_size:
return cls.languages_by_size
raise NotImplementedError(
'Family %s needs property "languages_by_size" or "codes"'
diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py
index 9a7f1ff..b0303d8 100644
--- a/pywikibot/tools/__init__.py
+++ b/pywikibot/tools/__init__.py
@@ -129,7 +129,7 @@
class classproperty(object): # noqa: N801
"""
- Metaclass to accesss a class method as a property.
+ Descriptor class to accesss a class method as a property.
This class may be used as a decorator::
@@ -147,6 +147,7 @@
def __init__(self, cls_method):
"""Hold the class method."""
self.method = cls_method
+ self.__doc__ = self.method.__doc__
def __get__(self, instance, owner):
"""Get the attribute of the owner class by its method."""
--
To view, visit https://gerrit.wikimedia.org/r/435676
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: I790580b8d5d9e6349a26ab52c8859a7148441bff
Gerrit-Change-Number: 435676
Gerrit-PatchSet: 5
Gerrit-Owner: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/434836 )
Change subject: [IMPR] Refine category tidy
......................................................................
[IMPR] Refine category tidy
+ list categories in two columns when acceptable to save some space
+ command line interface (output) improvements
+ fix category tidy edit summary, see:
https://cs.wikipedia.org/w/index.php?diff=prev&oldid=16104149
+ skip initial images, comments and templates when outputting beginning
of an article
+ sort categories to assign expectable numbers (when running tidy
on the same category again, the numbers changed until now)
+ doc improvements
+ naming convention improvements
Bug: T166476
Change-Id: I6792c4c19d5c235111165ad45982fe4257dde43c
---
M scripts/category.py
1 file changed, 205 insertions(+), 96 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/category.py b/scripts/category.py
index b4fe075..a6a5538 100755
--- a/scripts/category.py
+++ b/scripts/category.py
@@ -11,7 +11,7 @@
* add - mass-add a category to a list of pages
* remove - remove category tag from all pages in a category
* move - move all pages in a category to another category
- * tidy - tidy up a category by moving its articles into subcategories
+ * tidy - tidy up a category by moving its pages into subcategories
* tree - show a tree of subcategories of a given category
* listify - make a list of all of the articles that are in a category
@@ -121,10 +121,13 @@
from __future__ import absolute_import, unicode_literals
import codecs
+import math
import os
import pickle
import re
import sys
+
+from operator import methodcaller
import pywikibot
@@ -298,8 +301,9 @@
def _load(self):
if not self.is_loaded:
try:
- pywikibot.output(u'Reading dump from %s'
- % config.shortpath(self.filename))
+ if config.verbose_output:
+ pywikibot.output('Reading dump from %s'
+ % config.shortpath(self.filename))
with open_archive(self.filename, 'rb') as f:
databases = pickle.load(f)
# keys are categories, values are 2-tuples with lists as
@@ -950,35 +954,49 @@
class CategoryTidyRobot(Bot, CategoryPreprocess):
+ """
+ Robot to move members of a category into sub- or super-categories.
- """Script to help by moving members of the category into subcategories.
+ Specify the category title on the command line. The robot will
+ pick up the page, look for all sub- and super-categories, and show
+ them listed as possibilities to move page into with an assigned
+ number. It will ask you to type number of the appropriate
+ replacement, and performs the change robotically. It will then
+ automatically loop over all pages in the category.
- Specify the category name on the command line. The program will pick up the
- page, and look for all subcategories and supercategories, and show them
- with a number adjacent to them. It will then automatically loop over all
- pages in the category. It will ask you to type the number of the
- appropriate replacement, and perform the change robotically.
-
- If you don't want to move the member to a subcategory or supercategory,
+ If you don't want to move the member to a sub- or super-category,
but to another category, you can use the 'j' (jump) command.
- Typing 's' will leave the complete page unchanged.
+ By typing 's' you can leave the complete page unchanged.
- Typing '?' will show you the first few bytes of the current page, helping
- you to find out what the page is about and in which other categories it
- currently is.
+ By typing 'm' you can show more content of the current page,
+ helping you to find out what the page is about and in which other
+ categories it currently is.
+ @param cat_title: a title of the category to process
+ @type: str
+
+ @param cat_db: a CategoryDatabase object
+ @type: CategoryDatabase object
+
+ @param namespaces: namespaces to focus on
+ @type: iterable of pywikibot.Namespace
+
+ @param comment: a custom summary for edits
+ @type: str
"""
- def __init__(self, catTitle, catDB, namespaces=None):
+ def __init__(self, cat_title, cat_db, namespaces=None, comment=None):
"""Initializer."""
- self.catTitle = catTitle
- self.catDB = catDB
+ self.cat_title = cat_title
+ self.cat_db = cat_db
+ if comment:
+ self.edit_summary = comment
+ else:
+ self.template_vars = {'oldcat': cat_title}
+
site = pywikibot.Site()
- self.editSummary = i18n.twtranslate(site, 'category-changing',
- {'oldcat': catTitle,
- 'newcat': u''})
- self.cat = pywikibot.Category(site, catTitle)
+ self.cat = pywikibot.Category(site, cat_title)
super(CategoryTidyRobot, self).__init__(
generator=pagegenerators.PreloadingGenerator(
self.cat.articles(namespaces=namespaces)))
@@ -986,118 +1004,209 @@
@deprecated_args(article='member')
def move_to_category(self, member, original_cat, current_cat):
"""
- Ask if it should be moved to one of the subcategories.
+ Ask whether to move it to one of the sub- or super-categories.
- Given a page which is in category original_cat, ask the user if
- it should be moved to one of original_cat's subcategories.
+ Given a page in the original_cat category, ask the user whether
+ to move it to one of original_cat's sub- or super-categories.
Recursively run through subcategories' subcategories.
- NOTE: current_cat is only used for internal recursion. You should
- always use current_cat = original_cat.
+ NOTE: current_cat is only used for internal recursion. You
+ should always use current_cat = original_cat.
+
+ @param member: a page to process
+ @type: pywikibot.Page
+
+ @param original_cat: original category to replace
+ @type: pywikibot.Category
+
+ @param current_cat: a category which is questioned
+ @type: pywikibot.Category
"""
class CatContextOption(ContextOption):
-
- """An option to show more and more context."""
-
- def __init__(self):
- """Initializer."""
- super(CatContextOption, self).__init__(
- 'print first part of the page (longer and longer)', '?',
- full_text, contextLength, 500)
+ """An option to show more and more context and categories."""
def output_range(self, start, end):
- pywikibot.output('\n' + full_text[:end] + '\n')
+ """Output a section and categories from the text."""
+ pywikibot.output(self.text[start:end] + '…')
# if categories weren't visible, show them additionally
- # (maybe this should always be shown?)
if len(self.text) > end:
- pywikibot.output('')
- pywikibot.output('Original categories: ')
for cat in member.categories():
- pywikibot.output(u'* %s' % cat.title())
+ if cat != original_cat:
+ pywikibot.output(cat.title(asLink=True))
+ else:
+ pywikibot.output(color_format(
+ '{lightpurple}{0}{default}',
+ current_cat.title(asLink=True)))
- pywikibot.output(u'')
- # Show the title of the page where the link was found.
- # Highlight the title in purple.
+ class CatIntegerOption(IntegerOption):
+ """An option allowing a range of integers."""
+
+ def list_categories(self, cat_list, prefix=''):
+ """
+ Output categories in one or two columns.
+
+ Determine whether the list contains long or short
+ category titles and output category titles
+ as enumerated options.
+
+ @param cat_list: sorted iterable of category titles
+ to output
+ @type: iterable of str
+
+ @param prefix: a prefix to assigned number index
+ @type: str
+ """
+ # can we can output in two columns?
+ count = len(cat_list)
+ if count > 1 and len(max(cat_list, key=len)) <= 31:
+ new_column = math.ceil(count / 2)
+ else:
+ new_column = 0
+
+ # determine number format
+ if count > 9:
+ index = '%2d'
+ else:
+ index = '%d'
+
+ lines = []
+ for i, cat in enumerate(cat_list):
+ if new_column:
+ if i == new_column:
+ break
+ # columnify
+ i2 = i + new_column
+ if i2 < count:
+ lines.append('[{}{}] {:35}[{}{}] {}'.format(
+ prefix, index % i, cat,
+ prefix, index % i2, cat_list[i2]))
+ else:
+ lines.append('[{}{}] {}'.format(
+ prefix, index % i, cat))
+ else:
+ lines.append('[{}{}] {}'.format(
+ prefix, index % i, cat))
+
+ # output the result
+ for line in lines:
+ pywikibot.output(line)
+
+ # show the title of the page where the link was found.
+ pywikibot.output('')
pywikibot.output(color_format(
- 'Treating page {0}, '
- 'currently in {lightpurple}{1}{default}',
- member.title(asLink=True), current_cat.title()))
+ '>>> {lightpurple}{0}{default} <<<', member.title()))
- # Determine a reasonable amount of context to print
+ # determine a reasonable amount of context
try:
- full_text = member.get(get_redirect=True)
+ full_text = member.get()
except pywikibot.NoPage:
- pywikibot.output('Page %s not found.' % member.title())
+ pywikibot.output('Page {} not found.'.format(member.title()))
return
- try:
- contextLength = full_text.index('\n\n')
- except ValueError: # substring not found
- contextLength = 500
- if full_text.startswith(u'[['): # probably an image
- # Add extra paragraph.
- contextLength = full_text.find('\n\n', contextLength + 2)
- if contextLength > 1000 or contextLength < 0:
- contextLength = 500
- context_option = CatContextOption()
+ # skip initial templates, images and coments for articles
+ if member.namespace() == member.site.namespaces.MAIN:
+ excludes = ('template', 'file', 'comment')
+ regexes = textlib._get_regexes(excludes, member.site)
+ i = 0
+ while i < 3:
+ i = 0
+ for reg in regexes:
+ if reg.match(full_text):
+ full_text = reg.sub(r'', full_text, count=1).lstrip()
+ else:
+ i += 1
+
+ # output context
+ context_option = CatContextOption('show more context', 'm', full_text,
+ 500, 500)
context_option.output()
- # we need list to index the choice
- subcatlist = list(self.catDB.getSubcats(current_cat))
- supercatlist = list(self.catDB.getSupercats(current_cat))
+ # get super- and sub-categories
+ # sort them to assign expectable numbers
+ supercatlist = sorted(self.cat_db.getSupercats(current_cat),
+ key=methodcaller('title'))
+ subcatlist = sorted(self.cat_db.getSubcats(current_cat),
+ key=methodcaller('title'))
- if not subcatlist:
- pywikibot.output('This category has no subcategories.\n')
+ # show categories as possible choices with numbers
+ pywikibot.output('')
+
+ supercat_option = CatIntegerOption(0, len(supercatlist), 'u')
if not supercatlist:
- pywikibot.output('This category has no supercategories.\n')
- # show subcategories as possible choices (with numbers)
- for i, supercat in enumerate(supercatlist):
- # layout: we don't expect a cat to have more than 10 supercats
- pywikibot.output(u'u%d - Move up to %s' % (i, supercat.title()))
- for i, subcat in enumerate(subcatlist):
- # layout: we don't expect a cat to have more than 100 subcats
- pywikibot.output(u'%2d - Move down to %s' % (i, subcat.title()))
- options = (IntegerOption(0, len(supercatlist), 'u'),
- IntegerOption(0, len(subcatlist)),
- StandardOption('jump to another category', 'j'),
- StandardOption('skip this page', 's'),
- StandardOption('remove this category tag', 'r'),
+ pywikibot.output('This category has no supercategories.')
+ else:
+ pywikibot.output('Move up to category:')
+ cat_list = [cat.title(withNamespace=False) for cat in supercatlist]
+ supercat_option.list_categories(cat_list, 'u')
+
+ subcat_option = CatIntegerOption(0, len(subcatlist))
+ if not subcatlist:
+ pywikibot.output('This category has no subcategories.')
+ else:
+ pywikibot.output('Move down to category:')
+ cat_list = [cat.title(withNamespace=False) for cat in subcatlist]
+ subcat_option.list_categories(cat_list)
+
+ # show possible options for the user
+ pywikibot.output('')
+ options = (supercat_option,
+ subcat_option,
+ StandardOption(color_format(
+ 'save page to category {lightpurple}{0}{default}',
+ current_cat.title(withNamespace=False)), 'c'),
+ StandardOption('remove the category from page', 'r'),
+ StandardOption('skip page', 's'),
context_option,
- StandardOption('save category as "{0}"'
- .format(current_cat.title()), 'c'))
+ StandardOption('jump to custom category', 'j'),
+ )
choice = pywikibot.input_choice(color_format(
- 'Choice for page {lightpurple}{0}{default}:\n',
- member.title()), options, default='c')
+ 'Choice for page {lightpurple}{0}{default}:', member.title()),
+ options, default='c')
if choice == 'c':
- pywikibot.output(u'Saving category as %s' % current_cat.title())
+ pywikibot.output('Saving page to {}'.format(current_cat.title()))
if current_cat == original_cat:
pywikibot.output('No changes necessary.')
else:
+ if not self.edit_summary:
+ self.template_vars.update({
+ 'newcat': current_cat.title(asLink=True, textlink=True)
+ })
+ self.edit_summary = i18n.twtranslate(self.site,
+ 'category-replacing',
+ self.template_vars)
+ # change the category tag
member.change_category(original_cat, current_cat,
- summary=self.editSummary)
+ summary=self.edit_summary)
doc_page = self.determine_template_target(member)
if doc_page != member:
doc_page.change_category(original_cat, current_cat,
include=self.includeonly,
- summary=self.editSummary)
+ summary=self.edit_summary)
+
elif choice == 'j':
- newCatTitle = pywikibot.input('Please enter the category the '
- 'page should be moved to:',
- default=None) # require an answer
- newCat = pywikibot.Category(pywikibot.Link('Category:' +
- newCatTitle))
+ new_cat_title = pywikibot.input('Please enter the category '
+ 'the page should be moved to:',
+ default=None) # require an answer
+ new_cat = pywikibot.Category(pywikibot.Link('Category:' +
+ new_cat_title))
# recurse into chosen category
- self.move_to_category(member, original_cat, newCat)
+ self.move_to_category(member, original_cat, new_cat)
+
elif choice == 'r':
+ if not self.edit_summary:
+ self.edit_summary = i18n.twtranslate(self.site,
+ 'category-removing',
+ self.template_vars)
# remove the category tag
member.change_category(original_cat, None,
- summary=self.editSummary)
+ summary=self.edit_summary)
doc_page = self.determine_template_target(member)
if doc_page != member:
doc_page.change_category(original_cat, None,
include=self.includeonly,
- summary=self.editSummary)
+ summary=self.edit_summary)
+
elif choice != 's':
if choice[0] == 'u':
# recurse into supercategory
@@ -1112,12 +1221,11 @@
"""Cleanups after run operation."""
if not self._treat_counter:
pywikibot.output('There are no pages or files in category {}'
- .format(self.catTitle))
+ .format(self.cat_title))
def treat(self, page):
"""Process page."""
pywikibot.output('')
- pywikibot.output(u'=' * 67)
self.move_to_category(page, self.cat, self.cat)
@@ -1372,8 +1480,9 @@
move_together=move_together,
keep_sortkey=keep_sortkey)
elif action == 'tidy':
- catTitle = pywikibot.input(u'Which category do you want to tidy up?')
- bot = CategoryTidyRobot(catTitle, catDB, genFactory.namespaces)
+ cat_title = pywikibot.input('Which category do you want to tidy up?')
+ bot = CategoryTidyRobot(cat_title, catDB, genFactory.namespaces,
+ editSummary)
elif action == 'tree':
catTitle = pywikibot.input(
u'For which category do you want to create a tree view?')
--
To view, visit https://gerrit.wikimedia.org/r/434836
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: I6792c4c19d5c235111165ad45982fe4257dde43c
Gerrit-Change-Number: 434836
Gerrit-PatchSet: 9
Gerrit-Owner: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Framawiki <framawiki(a)tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/434457 )
Change subject: [IMPR] Adjust page counter
......................................................................
[IMPR] Adjust page counter
- If the bot treat method is called the page is treat counter is increased
even if the page is skipped. There are implementations in treat method
to skip a page in ExistingPageBot, CreatingPageBot, RedirectPageBot and
NoRedirectPageBot.
- Initially the init_page method was meant for that by raising SkipPageError.
But redefining init_page for the classes mentioned above could lead to a
breaking change.
- Therefore a new method "skip_page" is introduced. It is called by the run
method after init_page and continues the loop if it returns True. The
treat counter isn't increased in this case.
- Deprecate the usage of SkipPageError.
Change-Id: Id9885f7a2797a56786ce18ed7c9ace487742ef23
---
M pywikibot/bot.py
1 file changed, 52 insertions(+), 48 deletions(-)
Approvals:
Framawiki: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index e6a058b..0deb48e 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -95,6 +95,8 @@
from warnings import warn
import webbrowser
+from textwrap import fill
+
import pywikibot
from pywikibot import config2 as config
from pywikibot import daemonize
@@ -115,7 +117,7 @@
)
from pywikibot.logging import critical
from pywikibot.tools import (
- deprecated, deprecate_arg, deprecated_args, PY2,
+ deprecated, deprecate_arg, deprecated_args, issue_deprecation_warning, PY2,
)
from pywikibot.tools._logging import (
LoggingFormatter as _LoggingFormatter,
@@ -1400,14 +1402,22 @@
pywikibot.output("Script terminated by exception:\n")
pywikibot.exception()
+ def init_page(self, page):
+ """Initialize a page before treating.
+
+ Also used to set the arrange the current site. This is called before
+ skip_page and treat.
+ """
+ pass
+
+ def skip_page(self, page):
+ """Return whether treat should be skipped for the page."""
+ return False
+
def treat(self, page):
"""Process one page (Abstract method)."""
raise NotImplementedError('Method %s.treat() not implemented.'
% self.__class__.__name__)
-
- def init_page(self, page):
- """Return whether treat should be executed for the page."""
- pass
def setup(self):
"""Some inital setup before run operation starts.
@@ -1439,9 +1449,12 @@
self.setup()
try:
for page in self.generator:
+ # preprocessing of the page
try:
self.init_page(page)
except SkipPageError as e:
+ issue_deprecation_warning('Use of SkipPageError',
+ 'BaseBot.skip_page() method', 2)
pywikibot.warning('Skipped "{0}" due to: {1}'.format(
page, e.reason))
if PY2:
@@ -1450,6 +1463,8 @@
sys.exc_clear()
continue
+ if self.skip_page(page):
+ continue
# Process the page
self.treat(page)
@@ -1598,13 +1613,19 @@
self._site = value
def init_page(self, page):
- """Set site if not defined and return if it's on the defined site."""
+ """Set site if not defined."""
if not self._site:
self.site = page.site
- elif page.site != self.site:
- raise SkipPageError(page,
- 'The bot is on site "{0}" but the page on '
- 'site "{1}"'.format(self.site, page.site))
+
+ def skip_page(self, page):
+ """Skip page it's site is not on the defined site."""
+ if page.site != self.site:
+ pywikibot.warning(
+ fill('Skipped "{page}" due to: '
+ 'The bot is on site "{site}" but the page on '
+ 'site "{page.site}"'.format(site=self.site, page=page)))
+ return True
+ return super(SingleSiteBot, self).skip_page(page)
class MultipleSitesBot(BaseBot):
@@ -1735,20 +1756,14 @@
"""A CurrentPageBot class which only treats existing pages."""
- def treat(self, page):
+ def skip_page(self, page):
"""Treat page if it exists and handle NoPage from it."""
if not page.exists():
- pywikibot.warning('Page "{0}" does not exist on {1}.'.format(
- page.title(), page.site))
- return
- try:
- super(ExistingPageBot, self).treat(page)
- except pywikibot.NoPage as e:
- if e.page != page:
- raise
pywikibot.warning(
- 'During handling of page "{0}" on {1} a NoPage exception was '
- 'raised.'.format(page.title(), page.site))
+ 'Page "{page.title()}" does not exist on {page.site}.'
+ .format(page=page))
+ return True
+ return super(ExistingPageBot, self).skip_page(page)
class FollowRedirectPageBot(CurrentPageBot):
@@ -1766,53 +1781,42 @@
"""A CurrentPageBot class which only treats nonexistent pages."""
- def treat(self, page):
+ def skip_page(self, page):
"""Treat page if doesn't exist."""
if page.exists():
- pywikibot.warning('Page "{0}" does already exist on {1}.'.format(
- page.title(), page.site))
- return
- super(CreatingPageBot, self).treat(page)
+ pywikibot.warning(
+ 'Page "{page.title()}" does already exist on {page.site}.'
+ .format(page=page))
+ return True
+ return super(CreatingPageBot, self).skip_page(page)
class RedirectPageBot(CurrentPageBot):
"""A RedirectPageBot class which only treats redirects."""
- def treat(self, page):
+ def skip_page(self, page):
"""Treat only redirect pages and handle IsNotRedirectPage from it."""
if not page.isRedirectPage():
- pywikibot.warning('Page "{0}" on {1} is skipped because it is not '
- 'a redirect'.format(page.title(), page.site))
- return
- try:
- super(RedirectPageBot, self).treat(page)
- except pywikibot.IsNotRedirectPage as e:
- if e.page != page:
- raise
pywikibot.warning(
- 'During handling of page "{0}" on {1} a IsNotRedirectPage '
- 'exception was raised.'.format(page.title(), page.site))
+ 'Page "{page.title()}" on {page.site} is skipped because it is'
+ 'not a redirect'.format(page=page))
+ return True
+ return super(RedirectPageBot, self).skip_page(page)
class NoRedirectPageBot(CurrentPageBot):
"""A NoRedirectPageBot class which only treats non-redirects."""
- def treat(self, page):
+ def skip_page(self, page):
"""Treat only non-redirect pages and handle IsRedirectPage from it."""
if page.isRedirectPage():
- pywikibot.warning('Page "{0}" on {1} is skipped because it is a '
- 'redirect'.format(page.title(), page.site))
- return
- try:
- super(NoRedirectPageBot, self).treat(page)
- except pywikibot.IsRedirectPage as e:
- if e.page != page:
- raise
pywikibot.warning(
- 'During handling of page "{0}" on {1} a IsRedirectPage '
- 'exception was raised.'.format(page.title(), page.site))
+ 'Page "{page.title()}" on {page.site} is skipped because it is'
+ 'a redirect'.format(page=page))
+ return True
+ return super(NoRedirectPageBot, self).skip_page(page)
class WikidataBot(Bot, ExistingPageBot):
--
To view, visit https://gerrit.wikimedia.org/r/434457
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: Id9885f7a2797a56786ce18ed7c9ace487742ef23
Gerrit-Change-Number: 434457
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Framawiki <framawiki(a)tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>