Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789897 )

Change subject: [IMPR] Print counter statistic for all counters ......................................................................

[IMPR] Print counter statistic for all counters

BaseBot provides a counter which holds 'read', 'write' and 'skip' by default but other counters can be added easily.

- rewrite exit() method that all counter values are printed - increase 'read' counter before processing the page to add the 'read' counter first. BotPage.counter keeps the order of insertion (with Python 3.6+; the order of Python 3.5 is not deterministic) - add tests accordingly - add counters in touch.py when a page is touched or purged - use 'upload' counter in specialbots/_upload.py - Update BaseBot documentation

Bug: T307834 Change-Id: I567bae073e49eb3bde083b82a30e9f2a76044950 --- M docs/api_ref/pywikibot.rst M pywikibot/bot.py M pywikibot/specialbots/_upload.py M scripts/touch.py M tests/bot_tests.py 5 files changed, 80 insertions(+), 59 deletions(-)

Approvals: Xqt: Verified; Looks good to me, approved

diff --git a/docs/api_ref/pywikibot.rst b/docs/api_ref/pywikibot.rst index 02bd83a..cd8f47c 100644 --- a/docs/api_ref/pywikibot.rst +++ b/docs/api_ref/pywikibot.rst @@ -29,6 +29,7 @@ --------------------

.. automodule:: pywikibot.bot + :member-order: bysource

pywikibot.bot_choice module ---------------------------- diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 274e4bf..e343643 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -1174,16 +1174,14 @@ 'Option opt.bar is 4711' """

- # Handler configuration. - # Only the keys of the dict can be passed as init options - # The values are the default values - # Overwrite this in subclasses! - available_options = {} # type: Dict[str, Any] + """ Handler configuration attribute. + Only the keys of the dict can be passed as `__init__` options. + The values are the default values. Overwrite this in subclasses! + """

def __init__(self, **kwargs: Any) -> None: - """ - Only accept options defined in available_options. + """Only accept options defined in available_options.

:param kwargs: bot options """ @@ -1206,8 +1204,10 @@

class BaseBot(OptionHandler):

- """ - Generic Bot to be subclassed. + """Generic Bot to be subclassed. + + Only accepts `generator` and options defined in + :attr:`available_options`.

This class provides a :meth:`run` method for basic processing of a generator one page at a time. @@ -1225,13 +1225,12 @@ properties.

If the subclass does not set a generator, or does not override - :meth:`treat` or :meth:`run`, NotImplementedError is raised. + :meth:`treat` or :meth:`run`, `NotImplementedError` is raised.

For bot options handling refer :class:`OptionHandler` class above.

.. versionchanged:: 7.0 - A counter attribute is provided which is a `collections.Counter`; - The default counters are 'read', 'write' and 'skip'. + A :attr:`counter` instance variable is provided. """

use_disambigs = None # type: Optional[bool] @@ -1250,18 +1249,14 @@ .. versionadded:: 7.2 """

- # Handler configuration. - # The values are the default values - # Extend this in subclasses! - available_options = { 'always': False, # By default ask for confirmation when putting a page }

update_options = {} # type: Dict[str, Any] - """update_options can be used to update available_options; + """`update_options` can be used to update :attr:`available_options`; do not use it if the bot class is to be derived but use - self.available_options.update(<dict>) initializer in such case. + `self.available_options.update(<dict>)` initializer in such case.

.. versionadded:: 6.4 """ @@ -1269,7 +1264,7 @@ _current_page = None # type: Optional[pywikibot.page.BasePage]

def __init__(self, **kwargs: Any) -> None: - """Only accept 'generator' and options defined in available_options. + """Initializer.

:param kwargs: bot options :keyword generator: a :attr:`generator` processed by :meth:`run` method @@ -1279,14 +1274,26 @@ pywikibot.warn('{} has a generator already. Ignoring argument.' .format(self.__class__.__name__)) else: - #: generator processed by :meth:`run` method + #: instance variable to hold the generator processed by + #: :meth:`run` method self.generator = kwargs.pop('generator')

self.available_options.update(self.update_options) super().__init__(**kwargs)

- self.counter = Counter() + self.counter = Counter() # type: Counter + """Instance variable which holds counters. The default counters + are 'read', 'write' and 'skip'. You can use your own counters like:: + + self.counter['delete'] += 1 + + .. versionadded:: 7.0 + .. versionchanged:: 7.3 + Your additional counters are also printed during :meth:`exit` + """ + self._generator_completed = False + #: instance variable to hold the default page type self.treat_page_type = pywikibot.page.BasePage # type: Any

@@ -1378,21 +1385,18 @@ """ Save a new revision of a page, with user confirmation as required.

- Print differences, ask user for confirmation, - and puts the page if needed. + Print differences, ask user for confirmation, and puts the page + if needed.

Option used:

* 'always'

- Keyword args used: - - * 'asynchronous' - passed to page.save - * 'summary' - passed to page.save - * 'show_diff' - show changes between oldtext and newtext (enabled) - * 'ignore_save_related_errors' - report and ignore (disabled) - * 'ignore_server_errors' - report and ignore (disabled) - + :keyword asynchronous: passed to page.save + :keyword summary: passed to page.save + :keyword show_diff: show changes between oldtext and newtext (enabled) + :keyword ignore_save_related_errors: report and ignore (disabled) + :keyword ignore_server_errors: report and ignore (disabled) :return: whether the page was saved successfully """ if oldtext.rstrip() == newtext.rstrip(): @@ -1403,7 +1407,6 @@ self.current_page = page

show_diff = kwargs.pop('show_diff', True) - if show_diff: pywikibot.showDiff(oldtext, newtext)

@@ -1419,6 +1422,8 @@ """ Helper function to handle page save-related option error handling.

+ .. note:: Do no use it directly. Use :meth:`userPut` instead. + :param page: currently edited page :param func: the function to call :param args: passed to the function @@ -1430,6 +1435,8 @@ page save will be reported and ignored (default: False) :kwtype ignore_save_related_errors: bool :return: whether the page was saved successfully + + :meta public: """ if not self.user_confirm('Do you want to accept these changes?'): return False @@ -1473,13 +1480,17 @@ raise QuitKeyboardInterrupt

def exit(self) -> None: - """ - Cleanup and exit processing. + """Cleanup and exit processing.

- Invoked when Bot.run() is finished. - Prints treat and save counters and informs whether the script + Invoked when :meth:`run` is finished. Waits for pending threads, + prints counter statistics and informs whether the script terminated gracefully or was halted by exception. - May be overridden by subclasses. + + .. note:: Do not overwrite it by subclasses; :meth:`teardown` + should be used instead. + + .. versionchanged:: 7.3 + Statistics are printed for all entries in :attr:`counter` """ self.teardown() if hasattr(self, '_start_ts'): @@ -1489,10 +1500,10 @@ # wait until pending threads finished but don't close the queue pywikibot.stopme()

- pywikibot.output('\n{read} pages read' - '\n{write} pages written' - '\n{skip} pages skipped' - .format_map(self.counter)) + pywikibot.info() + for op, count in self.counter.items(): + pywikibot.info('{} {} operation{}' + .format(count, op, 's' if count > 1 else ''))

if hasattr(self, '_start_ts'): write_delta = pywikibot.Timestamp.now() - self._start_ts @@ -1508,10 +1519,12 @@ if self.counter['read']: pywikibot.output('Read operation time: {:.1f} seconds' .format(read_seconds / self.counter['read'])) - if self.counter['write']: - pywikibot.output( - 'Write operation time: {:.1f} seconds' - .format(write_seconds / self.counter['write'])) + + for op, count in self.counter.items(): + if not count or op == 'read': + continue + pywikibot.info('{} operation time: {:.1f} seconds' + .format(op.capitalize(), write_seconds / count))

# exc_info contains exception from self.run() while terminating exc_info = sys.exc_info() @@ -1525,13 +1538,15 @@ def init_page(self, item: Any) -> 'pywikibot.page.BasePage': """Initialize a generator item before treating.

- Ensure that the result of init_page is always a pywikibot.Page object - even when the generator returns something else. + Ensure that the result of `init_page` is always a + pywikibot.Page object or any other type given by the + :attr:`treat_page_type` even when the generator returns + something else.

- Also used to set the arrange the current site. This is called before - skip_page and treat. + Also used to set the arrange the current site. This is called + before :meth:`skip_page` and :meth:`treat`.

- :param item: any item from self.generator + :param item: any item from :attr:`generator` :return: return the page object to be processed further """ return item @@ -1576,17 +1591,18 @@ .format(self.__class__.__name__))

def setup(self) -> None: - """Some initial setup before run operation starts. + """Some initial setup before :meth:`run` operation starts.

This can be used for reading huge parts from life wiki or file operation which is more than just initialize the instance. - Invoked by run() before running through generator loop. + Invoked by :meth:`run` before running through :attr:`generator` + loop.

.. versionadded:: 3.0 """

def teardown(self) -> None: - """Some cleanups after run operation. Invoked by exit(). + """Some cleanups after :meth:`run` operation. Invoked by :meth:`exit`.

.. versionadded:: 3.0 """ @@ -1621,8 +1637,8 @@ continue

# Process the page - self.treat(page) self.counter['read'] += 1 + self.treat(page)

self._generator_completed = True except QuitKeyboardInterrupt: diff --git a/pywikibot/specialbots/_upload.py b/pywikibot/specialbots/_upload.py index 52409eb..743f38e 100644 --- a/pywikibot/specialbots/_upload.py +++ b/pywikibot/specialbots/_upload.py @@ -431,7 +431,7 @@ # No warning, upload complete. pywikibot.output('Upload of {} successful.' .format(filename)) - self.counter['write'] += 1 + self.counter['upload'] += 1 return filename # data['filename'] pywikibot.output('Upload aborted.') break diff --git a/scripts/touch.py b/scripts/touch.py index 292f8ad..48386dd 100755 --- a/scripts/touch.py +++ b/scripts/touch.py @@ -61,6 +61,8 @@ .format(page.title(as_link=True))) except PageSaveRelatedError as e: pywikibot.error('Page {} not saved:\n{}'.format(page, e.args)) + else: + self.counter['touch'] += 1

class PurgeBot(MultipleSitesBot): @@ -76,9 +78,11 @@

def treat(self, page) -> None: """Purge the given page.""" + done = page.purge(**self.opt) + if done: + self.counter['purge'] += 1 pywikibot.output('Page {}{} purged' - .format(page, - '' if page.purge(**self.opt) else ' not')) + .format(page, '' if done else ' not'))

def main(*args: str) -> None: diff --git a/tests/bot_tests.py b/tests/bot_tests.py index 12c89de..8b51092 100755 --- a/tests/bot_tests.py +++ b/tests/bot_tests.py @@ -195,7 +195,7 @@ pywikibot.Page(self.en, 'Page 2'), pywikibot.Page(self.de, 'Page 3')], post_treat) - self.bot.exit = self._exit(2, exception=ValueError) + self.bot.exit = self._exit(3, exception=ValueError) with self.assertRaisesRegex(ValueError, 'Whatever'): self.bot.run()

@@ -212,7 +212,7 @@ pywikibot.Page(self.de, 'Page 3')], post_treat)

- self.bot.exit = self._exit(2, exception=None) + self.bot.exit = self._exit(3, exception=None) self.bot.run()