jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1037886?usp=email )
Change subject: [IMPR] check whether BaseBot.generator is None in treat method ......................................................................
[IMPR] check whether BaseBot.generator is None in treat method
- check whether BaseBot.generator is None in treat method, call suggest_help in such case and leave the run method. - update documentation for BaseBot - remove suggest_help calls from several scripts where Bot.run() calls it already.
Change-Id: I935218ef1c1c18ff8b4f1219e5ffdb371613750f --- M docs/api_ref/bot.rst M pywikibot/bot.py M scripts/add_text.py M scripts/change_pagelang.py M scripts/claimit.py M scripts/clean_sandbox.py M scripts/coordinate_import.py M scripts/fixing_redirects.py M scripts/illustrate_wikidata.py M scripts/interwikidata.py M scripts/noreferences.py M scripts/replace.py M scripts/solve_disambiguation.py M tests/interwikidata_tests.py 14 files changed, 142 insertions(+), 90 deletions(-)
Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved
diff --git a/docs/api_ref/bot.rst b/docs/api_ref/bot.rst index 6901de0..640902f 100644 --- a/docs/api_ref/bot.rst +++ b/docs/api_ref/bot.rst @@ -5,3 +5,18 @@ .. automodule:: bot :synopsis: User-interface related functions for building bots :member-order: bysource + + .. autoclass:: BaseBot + + .. attribute:: generator + :type: Iterable + + Instance variable to hold the Iterbale processed by :meth:`run` + method. The is added to the class with *generator* keyword + argument and the proposed type is a ``Generator``. If not, + :meth:`run` upcast the generator attribute to become a + ``Generator`` type. If a :class:`BaseBot` subclass has its own + ``generator`` attribute, a warning will be thrown when an + object is passed to *generator* keyword parameter. + + .. warning:: this is just a sample diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 935886d..bb732d8 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -1069,17 +1069,15 @@ This class provides a :meth:`run` method for basic processing of a generator one page at a time.
- If the subclass places a page generator in - :attr:`self.generator<generator>`, Bot will process each page in the - generator, invoking the method :meth:`treat` which must then be - implemented by subclasses. + If the subclass places a page generator in :attr:`generator`, Bot + will process each page in the generator, invoking the method + :meth:`treat` which must then be implemented by subclasses.
- Each item processed by :meth:`treat` must be a - :class:`page.BasePage` type. Use :meth:`init_page` to - upcast the type. To enable other types, set - :attr:`BaseBot.treat_page_type` to an appropriate type; your bot - should derive from :class:`BaseBot` in that case and handle site - properties. + Each item processed by :meth:`treat` must be a :class:`page.BasePage` + type. Use :meth:`init_page` to upcast the type. To enable other + types, set :attr:`BaseBot.treat_page_type` to an appropriate type; + your bot should derive from :class:`BaseBot` in that case and handle + site properties.
If the subclass does not set a generator, or does not override :meth:`treat` or :meth:`run`, `NotImplementedError` is raised. @@ -1133,15 +1131,14 @@ """Initializer.
:param kwargs: bot options - :keyword generator: a :attr:`generator` processed by :meth:`run` method + :keyword generator: a :attr:`generator` processed by :meth:`run` + method """ if 'generator' in kwargs: if hasattr(self, 'generator'): - pywikibot.warn('{} has a generator already. Ignoring argument.' - .format(self.__class__.__name__)) + pywikibot.warn(f'{type(self).__name__} has a generator' + ' already. Ignoring argument.') else: - #: instance variable to hold the generator processed by - #: :meth:`run` method self.generator: Iterable = kwargs.pop('generator')
self.available_options.update(self.update_options) @@ -1149,7 +1146,8 @@
self.counter: Counter = Counter() """Instance variable which holds counters. The default counters - are 'read', 'write' and 'skip'. You can use your own counters like:: + are 'read', 'write' and 'skip'. All of them are printed within + :meth:`exit`. You can use your own counters like::
self.counter['delete'] += 1
@@ -1159,21 +1157,30 @@ """
self.generator_completed: bool = False - """Instance attribute which is True if the generator is completed. + """ + Instance attribute which is True if the :attr:`generator` is completed. + + It gives False if the the generator processing in :meth:`run` is + either interrupted by ``KeyboardInterrupt`` or exited by + :exc:`QuitKeyboardInterrupt` while closing the generator i.e. + :code:`self.generator.close()` keeps the value True.
To check for an empty generator you may use::
if self.generator_completed and not self.counter['read']: print('generator was emtpty')
- .. note:: An empty generator remains False. + .. note:: An empty generator returns True. .. versionadded:: 3.0 .. versionchanged:: 7.4 renamed to `generator_completed` to become a public attribute. """
- #: instance variable to hold the default page type self.treat_page_type: Any = pywikibot.page.BasePage + """Instance variable to hold the default page type used by :meth:`run`. + + .. versionadded:: 6.1 + """
@property def current_page(self) -> pywikibot.page.BasePage: @@ -1455,19 +1462,86 @@ def run(self) -> None: """Process all pages in generator.
- :raise AssertionError: "page" is not a pywikibot.page.BasePage object + Call :meth:`setup`, check for a valid ``Iterable`` type in + :attr:`generator`, upcast it to a ``Generator`` type if + necessary, process every generator`s item as follows: + + For each item call :meth:`init_page`, check whether the result + is a :attr:`treat_page_type` type, call :meth:`skip_page` to + determine whether to skip the current page. Otherwise call + :meth:`treat` for each item. + + This method also adjust ``read`` and ``skip`` :attr:`counter`, + and finally it calls :meth:`exit` when leaving the method. In + short this method is implemented similar to this: + + .. code-block:: python + + def run(self) -> None: + '''Process all pages in generator.''' + self.setup() + + if not hasattr(self, 'generator'): + raise NotImplementedError('"generator" not set.') + + if self.generator is None; + print('No generator was defined') + + try: + for item in self.generator: + page = self.init_page(item) + + if self.skip_page(page): + continue + + self.treat(page) + + except(QuitKeyboardInterrupt, KeyboardInterrupt): + print('User canceled bot run.') + + finally: + self.exit() + + .. versionchanged:: 3.0 + ``skip`` counter was added.; call :meth:`setup` first. + .. versionchanged:: 6.0 + upcast :attr:`generator` to a ``Generator`` type to enable + ``generator.close()`` method. + .. versionchanged:: 6.1 + Objects from :attr:`generator` may be different from + :class:`pywikibot.Page` but the type must be registered in + :attr:`treat_page_type`. + .. versionchanged:: 9.2 + leave method gracefully if :attr:`generator` is None using + :func:`suggest_help` function. + + :raise AssertionError: "page" is not a pywikibot.page.BasePage + object + :raise KeyboardInterrupt: KeyboardInterrupt occurred while + :attr:`config.verbose_output` was set + :raise NotImplementedError: :attr:`generator` is not set + :raise TypeError: invalid generator type or page is not a + :attr:`treat_page_type` """ self._start_ts = pywikibot.Timestamp.now() self.setup()
if not hasattr(self, 'generator'): - raise NotImplementedError('Variable {}.generator not set.' - .format(self.__class__.__name__)) + raise NotImplementedError( + f'Variable {type(self).__name__}.generator not set.') + + if suggest_help(missing_generator=self.generator is None): + return + if not isinstance(self.generator, Generator): - # to provide close() method - pywikibot.debug('wrapping {} type to a Generator type' - .format(type(self.generator).__name__)) - self.generator = (item for item in self.generator) + gen_type = type(self.generator).__name__ + pywikibot.debug(f'wrapping {gen_type} type to a Generator type') + try: + # to provide generator.close() method + self.generator = (item for item in self.generator) + except TypeError: + raise TypeError(f'Invalid type {gen_type} for generator') + try: for item in self.generator: # preprocessing of the page @@ -1475,9 +1549,8 @@
# validate page type if not isinstance(page, self.treat_page_type): - raise TypeError('"page" is not a {!r} object but {}.' - .format(self.treat_page_type, - page.__class__.__name__)) + raise TypeError(f'"page" is not a {self.treat_page_type!r}' + f' object but {type(page).__name__}.')
if self.skip_page(page): self.counter['skip'] += 1 @@ -1489,13 +1562,13 @@
self.generator_completed = True except QuitKeyboardInterrupt: - pywikibot.info(f'\nUser quit {self.__class__.__name__} bot run...') + pywikibot.info(f'\nUser quit {type(self).__name__} bot run...') except KeyboardInterrupt: if config.verbose_output: raise
- pywikibot.info('\nKeyboardInterrupt during {} bot run...' - .format(self.__class__.__name__)) + pywikibot.info( + f'\nKeyboardInterrupt during {type(self).__name__} bot run...') finally: self.exit()
diff --git a/scripts/add_text.py b/scripts/add_text.py index d5c07bd..277511a 100755 --- a/scripts/add_text.py +++ b/scripts/add_text.py @@ -185,9 +185,6 @@ return
generator = generator_factory.getCombinedGenerator() - if pywikibot.bot.suggest_help(missing_generator=not generator): - return - bot = AddTextBot(generator=generator, **options) bot.run()
diff --git a/scripts/change_pagelang.py b/scripts/change_pagelang.py index 473b785..fc7ee82 100755 --- a/scripts/change_pagelang.py +++ b/scripts/change_pagelang.py @@ -182,11 +182,8 @@ return
gen = gen_factory.getCombinedGenerator(preload=True) - if gen: - bot = ChangeLangBot(generator=gen, **options) - bot.run() - else: - pywikibot.bot.suggest_help(missing_generator=True) + bot = ChangeLangBot(generator=gen, **options) + bot.run()
if __name__ == '__main__': diff --git a/scripts/claimit.py b/scripts/claimit.py index 11ecb4c..ab892fc 100755 --- a/scripts/claimit.py +++ b/scripts/claimit.py @@ -46,7 +46,7 @@
""" # -# (C) Pywikibot team, 2013-2023 +# (C) Pywikibot team, 2013-2024 # # Distributed under the terms of the MIT license. # @@ -150,10 +150,6 @@ claims.append(claim)
generator = gen.getCombinedGenerator() - if not generator: - pywikibot.bot.suggest_help(missing_generator=True) - return - bot = ClaimRobot(claims, exists_arg, generator=generator) bot.run()
diff --git a/scripts/clean_sandbox.py b/scripts/clean_sandbox.py index c0ff4d5..d48f4c3 100755 --- a/scripts/clean_sandbox.py +++ b/scripts/clean_sandbox.py @@ -214,15 +214,17 @@ if not self.translated_content: raise RuntimeError( 'No content is given for sandbox pages, exiting.') + if not self.generator: pages = [] for item in sandbox_titles: p = self.site.page_from_repository(item) if p is not None: pages.append(p) - if not pages: - pywikibot.bot.suggest_help(missing_generator=True) + + if pywikibot.bot.suggest_help(missing_generator=not pages): sys.exit() + self.generator = pages
def run(self) -> None: diff --git a/scripts/coordinate_import.py b/scripts/coordinate_import.py index e517afa..de4f6d2 100755 --- a/scripts/coordinate_import.py +++ b/scripts/coordinate_import.py @@ -178,12 +178,8 @@ # FIXME: this preloading preloads neither coordinates nor Wikibase items # but preloads wikitext which we don't need generator = generator_factory.getCombinedGenerator(preload=True) - - if generator: - coordbot = CoordImportRobot(generator=generator, create=create_new) - coordbot.run() - else: - pywikibot.bot.suggest_help(missing_generator=True) + coordbot = CoordImportRobot(generator=generator, create=create_new) + coordbot.run()
if __name__ == '__main__': diff --git a/scripts/fixing_redirects.py b/scripts/fixing_redirects.py index 9b52cd7..4774b1a 100755 --- a/scripts/fixing_redirects.py +++ b/scripts/fixing_redirects.py @@ -17,7 +17,7 @@ ¶ms; """ # -# (C) Pywikibot team, 2004-2023 +# (C) Pywikibot team, 2004-2024 # # Distributed under the terms of the MIT license. # @@ -256,11 +256,9 @@ return else: gen = gen_factory.getCombinedGenerator(preload=True) - if gen: - bot = FixingRedirectBot(generator=gen, **options) - bot.run() - else: - suggest_help(missing_generator=True) + + bot = FixingRedirectBot(generator=gen, **options) + bot.run()
if __name__ == '__main__': diff --git a/scripts/illustrate_wikidata.py b/scripts/illustrate_wikidata.py index 220bd51..bf6730f 100755 --- a/scripts/illustrate_wikidata.py +++ b/scripts/illustrate_wikidata.py @@ -13,7 +13,7 @@ ¶ms; """ # -# (C) Pywikibot team, 2013-2022 +# (C) Pywikibot team, 2013-2024 # # Distributed under the terms of MIT license. # @@ -100,10 +100,6 @@ generator_factory.handle_arg(arg)
generator = generator_factory.getCombinedGenerator(preload=True) - if not generator: - pywikibot.bot.suggest_help(missing_generator=True) - return - bot = IllustrateRobot(wdproperty, generator=generator) bot.run()
diff --git a/scripts/interwikidata.py b/scripts/interwikidata.py index f31a66e..74bd226 100755 --- a/scripts/interwikidata.py +++ b/scripts/interwikidata.py @@ -39,12 +39,7 @@ import pywikibot.i18n import pywikibot.textlib from pywikibot import info, pagegenerators, warning -from pywikibot.bot import ( - ConfigParserBot, - ExistingPageBot, - SingleSiteBot, - suggest_help, -) +from pywikibot.bot import ConfigParserBot, ExistingPageBot, SingleSiteBot from pywikibot.exceptions import APIError, NoPageError
@@ -262,11 +257,8 @@ site = pywikibot.Site()
generator = gen_factory.getCombinedGenerator(preload=True) - if generator: - bot = IWBot(generator=generator, site=site, **options) - bot.run() - else: - suggest_help(missing_generator=True) + bot = IWBot(generator=generator, site=site, **options) + bot.run()
if __name__ == '__main__': diff --git a/scripts/noreferences.py b/scripts/noreferences.py index b4164a3..7e30984 100755 --- a/scripts/noreferences.py +++ b/scripts/noreferences.py @@ -838,11 +838,8 @@ if cat: gen = cat.articles(namespaces=genFactory.namespaces or [0])
- if gen: - bot = NoReferencesBot(generator=gen, **options) - bot.run() - else: - pywikibot.bot.suggest_help(missing_generator=True) + bot = NoReferencesBot(generator=gen, **options) + bot.run()
if __name__ == '__main__': diff --git a/scripts/replace.py b/scripts/replace.py index cb075d9..a1786d1 100755 --- a/scripts/replace.py +++ b/scripts/replace.py @@ -1137,9 +1137,6 @@ gen = handle_sql(sql_query, replacements, exceptions['text-contains'])
gen = genFactory.getCombinedGenerator(gen, preload=preload) - if pywikibot.bot.suggest_help(missing_generator=not gen): - return - bot = ReplaceRobot(gen, replacements, exceptions, site=site, summary=edit_summary, **options) site.login() diff --git a/scripts/solve_disambiguation.py b/scripts/solve_disambiguation.py index 878e048..f0173ef 100755 --- a/scripts/solve_disambiguation.py +++ b/scripts/solve_disambiguation.py @@ -1283,10 +1283,6 @@ generator_factory.handle_arg(argument)
generator = generator_factory.getCombinedGenerator(generator) - if not generator: - pywikibot.bot.suggest_help(missing_generator=True) - return - bot = DisambiguationRobot(generator=generator, pos=alternatives, **options) bot.run()
diff --git a/tests/interwikidata_tests.py b/tests/interwikidata_tests.py index 45ced29..e1bfb08 100755 --- a/tests/interwikidata_tests.py +++ b/tests/interwikidata_tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Tests for scripts/interwikidata.py.""" # -# (C) Pywikibot team, 2015-2022 +# (C) Pywikibot team, 2015-2024 # # Distributed under the terms of the MIT license. # @@ -55,9 +55,9 @@
def test_main(self): """Test main function interwikidata.py.""" - # The main function should return False when no generator is defined. + # The main function return None. with empty_sites(): - self.assertFalse(interwikidata.main()) + self.assertIsNone(interwikidata.main())
def test_iw_bot(self): """Test IWBot class."""