jenkins-bot submitted this change.
[tests] Suppress and check replacebot logs
Our unit tests dump ~536 lines to stdout because bots print their logs. This
verbosity makes it difficult to glean important information from our test
output.
This adds a buffer interface to capture bot logs and let us assert about
their content. This has a couple advantages...
1. Less verbose test output.
2. Better test coverage by ensuring our bots emit the right output.
If you like this direction I'll do the same for our other tests.
Bug: T281199
Change-Id: I2c204795e9ff8189f95eebf658f3aeffb741aba0
---
M pywikibot/bot.py
M pywikibot/config.py
A pywikibot/userinterfaces/buffer_interface.py
M tests/aspects.py
M tests/diff_tests.py
M tests/i18n_tests.py
M tests/replacebot_tests.py
M tests/ui_tests.py
8 files changed, 203 insertions(+), 57 deletions(-)
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 7dc13bd..087a2bb 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -176,13 +176,8 @@
# Note: all output goes through python std library "logging" module
_logger = 'bot'
-# User interface initialization
-# search for user interface module in the 'userinterfaces' subdirectory
-uiModule = __import__('pywikibot.userinterfaces.{}_interface'
- .format(config.userinterface), fromlist=['UI'])
-ui = uiModule.UI()
-atexit.register(ui.flush)
-pywikibot.argvu = ui.argvu()
+ui = None
+
_GLOBAL_HELP = """
GLOBAL OPTIONS
@@ -258,6 +253,24 @@
"""
+def set_interface(module_name):
+ """Configures any bots to use the given interface module."""
+ global ui
+
+ # User interface initialization
+ # search for user interface module in the 'userinterfaces' subdirectory
+ ui_module = __import__('pywikibot.userinterfaces.{}_interface'
+ .format(module_name), fromlist=['UI'])
+ ui = ui_module.UI()
+ atexit.register(ui.flush)
+ pywikibot.argvu = ui.argvu()
+
+ # re-initialize if we were already initialized with another UI
+
+ if _handlers_initialized:
+ init_handlers()
+
+
# Initialize the handlers and formatters for the logging system.
#
# This relies on the global variable 'ui' which is a UserInterface object
@@ -2229,3 +2242,6 @@
raise NotImplementedError('Method {}.treat_page_and_item() not '
'implemented.'
.format(self.__class__.__name__))
+
+
+set_interface(config.userinterface)
diff --git a/pywikibot/config.py b/pywikibot/config.py
index d263da0..2304599 100644
--- a/pywikibot/config.py
+++ b/pywikibot/config.py
@@ -472,7 +472,7 @@
# https://docs.python.org/3/library/codecs.html#standard-encodings
textfile_encoding = 'utf-8'
-# currently terminal is the only userinterface supported
+# currently terminal and buffer are the only supported userinterfaces
userinterface = 'terminal'
# this can be used to pass variables to the UI init function
diff --git a/pywikibot/userinterfaces/buffer_interface.py b/pywikibot/userinterfaces/buffer_interface.py
new file mode 100755
index 0000000..8978304
--- /dev/null
+++ b/pywikibot/userinterfaces/buffer_interface.py
@@ -0,0 +1,68 @@
+"""Non-interactive interface that stores output."""
+#
+# (C) Pywikibot team, 2021
+#
+# Distributed under the terms of the MIT license.
+#
+import logging
+from typing import Any, Sequence, Union
+
+from pywikibot import config
+from pywikibot.logging import INFO, VERBOSE
+from pywikibot.userinterfaces._interface_base import ABUIC
+
+
+class UI(ABUIC, logging.Handler):
+
+ """Collects output into an unseen buffer."""
+
+ def __init__(self):
+ """Initialize the UI."""
+ super().__init__()
+ self.setLevel(VERBOSE if config.verbose_output else INFO)
+ self.setFormatter(logging.Formatter(fmt='%(message)s%(newline)s'))
+
+ self._output = []
+
+ def init_handlers(self, root_logger, *args, **kwargs):
+ """Initialize the handlers for user output."""
+ root_logger.addHandler(self)
+
+ def input(self, question: str, password: bool = False,
+ default: str = '', force: bool = False) -> str:
+ """Ask the user a question and return the answer."""
+ return default
+
+ def input_choice(self, question: str, options, default: str = None,
+ return_shortcut: bool = True,
+ automatic_quit: bool = True, force: bool = False):
+ """Ask the user and returns a value from the options."""
+ return default
+
+ def input_list_choice(self, question: str, answers: Sequence[Any],
+ default: Union[int, str, None] = None,
+ force: bool = False) -> Any:
+ """Ask the user to select one entry from a list of entries."""
+ return default
+
+ def output(self, text, *args, **kwargs) -> None:
+ """Output text that would usually go to a stream."""
+ self._output.append(text)
+
+ def emit(self, record: logging.LogRecord) -> None:
+ """Logger output."""
+ self.output(record.getMessage())
+
+ def get_output(self):
+ """Provides any output we've buffered."""
+ return list(self._output)
+
+ def pop_output(self):
+ """Provide and clear any buffered output."""
+ buffered_output = self.get_output()
+ self.clear()
+ return buffered_output
+
+ def clear(self):
+ """Removes any buffered output."""
+ self._output.clear()
diff --git a/tests/aspects.py b/tests/aspects.py
index 8a137b0..98b44cb 100644
--- a/tests/aspects.py
+++ b/tests/aspects.py
@@ -62,6 +62,7 @@
return f
OSWIN32 = (sys.platform == 'win32')
+pywikibot.bot.set_interface('buffer')
class TestCaseBase(unittest.TestCase):
diff --git a/tests/diff_tests.py b/tests/diff_tests.py
index 1808947..6fdea61 100644
--- a/tests/diff_tests.py
+++ b/tests/diff_tests.py
@@ -218,7 +218,7 @@
mock.assert_any_call(self.header_base.format(header))
@patch('pywikibot.output')
- @patch('pywikibot.userinterfaces.terminal_interface_base.UI.input',
+ @patch('pywikibot.userinterfaces.buffer_interface.UI.input',
return_value='y')
def test_accept(self, input, mock):
"""Check output of cherry_pick if changes accepted."""
@@ -227,7 +227,7 @@
mock.assert_any_call(self.diff_message)
@patch('pywikibot.output')
- @patch('pywikibot.userinterfaces.terminal_interface_base.UI.input',
+ @patch('pywikibot.userinterfaces.buffer_interface.UI.input',
return_value='n')
def test_reject(self, input, mock):
"""Check output of cherry_pick if changes rejected."""
@@ -237,7 +237,7 @@
mock.assert_any_call(self.none_message)
@patch('pywikibot.output')
- @patch('pywikibot.userinterfaces.terminal_interface_base.UI.input',
+ @patch('pywikibot.userinterfaces.buffer_interface.UI.input',
return_value='q')
def test_quit(self, input, mock):
"""Check output of cherry_pick if quit."""
@@ -247,7 +247,7 @@
mock.assert_any_call(self.none_message)
@patch('pywikibot.output')
- @patch('pywikibot.userinterfaces.terminal_interface_base.UI.input',
+ @patch('pywikibot.userinterfaces.buffer_interface.UI.input',
return_value='y')
def test_by_letter_accept(self, input, mock):
"""Check cherry_pick output.
@@ -260,7 +260,7 @@
mock.assert_any_call(self.diff_by_letter_message)
@patch('pywikibot.output')
- @patch('pywikibot.userinterfaces.terminal_interface_base.UI.input',
+ @patch('pywikibot.userinterfaces.buffer_interface.UI.input',
return_value='q')
def test_by_letter_quit(self, input, mock):
"""Check cherry_pick output.
diff --git a/tests/i18n_tests.py b/tests/i18n_tests.py
index 74c2be5..21d22d3 100644
--- a/tests/i18n_tests.py
+++ b/tests/i18n_tests.py
@@ -338,6 +338,7 @@
def setUp(self):
"""Patch the output and input methods."""
super().setUp()
+ bot.set_interface('terminal')
self.output_text = ''
self.orig_raw_input = bot.ui._raw_input
self.orig_output = bot.ui.output
@@ -350,6 +351,7 @@
config.cosmetic_changes_mylang_only = self.old_cc_setting
bot.ui._raw_input = self.orig_raw_input
bot.ui.output = self.orig_output
+ bot.set_interface('buffer')
super().tearDown()
def test_i18n_input(self):
diff --git a/tests/replacebot_tests.py b/tests/replacebot_tests.py
index d45bb73..863ab1f 100644
--- a/tests/replacebot_tests.py
+++ b/tests/replacebot_tests.py
@@ -71,6 +71,8 @@
replace.pywikibot.input = self._fake_input
replace.pywikibot.Site = patched_site
+ pywikibot.bot.ui.clear()
+
def tearDown(self):
"""Bring back the old bot class."""
replace.ReplaceRobot = self._original_bot
@@ -99,9 +101,17 @@
"""Test invalid command line replacement configurations."""
# old and new need to be together
self.assertFalse(self._run('foo', '-pairsfile:/dev/null', 'bar'))
+
+ self.assertEqual([
+ '-pairsfile used between a pattern replacement pair.',
+ ], pywikibot.bot.ui.pop_output())
+
# only old provided
with empty_sites():
self.assertFalse(self._run('foo'))
+ self.assertEqual([
+ 'Incomplete command line pattern replacement pair.',
+ ], pywikibot.bot.ui.pop_output())
# In the end no bots should've been created
self.assertFalse(self.bots)
@@ -169,6 +179,11 @@
self.assertLength(bot.replacements, 1)
self._test_replacement(bot.replacements[0])
+ self.assertEqual([
+ 'The summary message for the command line replacements will '
+ 'be something like: Bot: Automated text replacement (-1 +2)',
+ ], pywikibot.bot.ui.pop_output())
+
def test_cmd_automatic(self):
"""Test command line replacements with automatic summary."""
bot = self._get_bot(None, '1', '2', '-automaticsummary')
@@ -176,17 +191,24 @@
self._test_replacement(bot.replacements[0])
self.assertEqual(self.inputs, [])
+ self.assertEqual([
+ 'The summary message for the command line replacements will '
+ 'be something like: Bot: Automated text replacement (-1 +2)',
+ ], pywikibot.bot.ui.pop_output())
+
def test_only_fix_global_message(self):
"""Test fixes replacements only."""
bot = self._get_bot(None, '-fix:has-msg')
self.assertLength(bot.replacements, 1)
self._test_fix_replacement(bot.replacements[0])
+ self.assertEqual([], pywikibot.bot.ui.pop_output())
def test_only_fix_global_message_tw(self):
"""Test fixes replacements only."""
bot = self._get_bot(None, '-fix:has-msg-tw')
self.assertLength(bot.replacements, 1)
self._test_fix_replacement(bot.replacements[0])
+ self.assertEqual([], pywikibot.bot.ui.pop_output())
def test_only_fix_no_message(self):
"""Test fixes replacements only."""
@@ -194,11 +216,18 @@
self.assertLength(bot.replacements, 1)
self._test_fix_replacement(bot.replacements[0])
+ self.assertEqual([
+ 'The summary will not be used when the fix has one defined but '
+ 'the following fix(es) do(es) not have a summary defined: '
+ '"no-msg" (all replacements)',
+ ], pywikibot.bot.ui.pop_output())
+
def test_only_fix_all_replacement_summary(self):
"""Test fixes replacements only."""
bot = self._get_bot(None, '-fix:all-repl-msg')
self.assertLength(bot.replacements, 1)
self._test_fix_replacement(bot.replacements[0], msg=True)
+ self.assertEqual([], pywikibot.bot.ui.pop_output())
def test_only_fix_partial_replacement_summary(self):
"""Test fixes replacements only."""
@@ -207,12 +236,19 @@
self._test_fix_replacement(replacement, 2, offset, offset == 0)
self.assertLength(bot.replacements, 2)
+ self.assertEqual([
+ 'The summary will not be used when the fix has one defined but '
+ 'the following fix(es) do(es) not have a summary defined: '
+ '"partial-repl-msg" (replacement #2)',
+ ], pywikibot.bot.ui.pop_output())
+
def test_only_fix_multiple(self):
"""Test fixes replacements only."""
bot = self._get_bot(None, '-fix:has-msg-multiple')
for offset, replacement in enumerate(bot.replacements):
self._test_fix_replacement(replacement, 3, offset)
self.assertLength(bot.replacements, 3)
+ self.assertEqual([], pywikibot.bot.ui.pop_output())
def test_cmd_and_fix(self):
"""Test command line and fix replacements together."""
@@ -221,6 +257,11 @@
self._test_replacement(bot.replacements[0])
self._test_fix_replacement(bot.replacements[1])
+ self.assertEqual([
+ 'The summary message for the command line replacements will be '
+ 'something like: Bot: Automated text replacement (-1 +2)',
+ ], pywikibot.bot.ui.pop_output())
+
def test_except_title(self):
"""Test excepting and requiring a title specific to fix."""
bot = self._get_bot(True, '-fix:no-msg-title-exceptions')
@@ -228,9 +269,27 @@
self._test_fix_replacement(bot.replacements[0])
self.assertIn('title', bot.replacements[0].exceptions)
self.assertIn('require-title', bot.replacements[0].exceptions)
+
+ self.assertEqual([
+ 'The summary will not be used when the fix has one defined but '
+ 'the following fix(es) do(es) not have a summary defined: '
+ '"no-msg-title-exceptions" (all replacements)',
+ ], pywikibot.bot.ui.pop_output())
+
self._apply(bot, 'Hello 1', missing=True, title='Neither')
+ self.assertEqual([
+ 'Skipping fix "no-msg-title-exceptions" on [[Neither]] because '
+ 'the title is on the exceptions list.',
+ ], pywikibot.bot.ui.pop_output())
+
self._apply(bot, 'Hello 2', title='Allowed')
+ self.assertEqual([], pywikibot.bot.ui.pop_output())
+
self._apply(bot, 'Hello 1', missing=True, title='Allowed Declined')
+ self.assertEqual([
+ 'Skipping fix "no-msg-title-exceptions" on [[Allowed Declined]] '
+ 'because the title is on the exceptions list.'
+ ], pywikibot.bot.ui.pop_output())
def test_fix_callable(self):
"""Test fix replacements using a callable."""
@@ -239,6 +298,12 @@
self._test_fix_replacement(bot.replacements[0])
self.assertTrue(callable(bot.replacements[0].new))
+ self.assertEqual([
+ 'The summary will not be used when the fix has one defined but '
+ 'the following fix(es) do(es) not have a summary defined: '
+ '"no-msg-callable" (all replacements)',
+ ], pywikibot.bot.ui.pop_output())
+
if __name__ == '__main__': # pragma: no cover
with suppress(SystemExit):
diff --git a/tests/ui_tests.py b/tests/ui_tests.py
index ac5c420..07e6974 100644
--- a/tests/ui_tests.py
+++ b/tests/ui_tests.py
@@ -22,7 +22,6 @@
STDOUT,
VERBOSE,
WARNING,
- ui,
)
from pywikibot.userinterfaces import (
terminal_interface_base,
@@ -60,21 +59,6 @@
self._stream.seek(0)
-def patched_print(text, target_stream):
- try:
- stream = patched_streams[target_stream]
- except KeyError:
- assert isinstance(target_stream,
- pywikibot.userinterfaces.win32_unicode.UnicodeOutput)
- assert target_stream._stream
- stream = patched_streams[target_stream._stream]
- org_print(text, stream)
-
-
-def patched_input():
- return strin._stream.readline().strip()
-
-
patched_streams = {}
strout = Stream('out', patched_streams)
strerr = Stream('err', patched_streams)
@@ -84,25 +68,6 @@
newstderr = strerr._stream
newstdin = strin._stream
-org_print = ui._print
-org_input = ui._raw_input
-
-
-def patch():
- """Patch standard terminal files."""
- strout.reset()
- strerr.reset()
- strin.reset()
- ui._print = patched_print
- ui._raw_input = patched_input
-
-
-def unpatch():
- """Un-patch standard terminal files."""
- ui._print = org_print
- ui._raw_input = org_input
-
-
logger = logging.getLogger('pywiki')
loggingcontext = {'caller_name': 'ui_tests',
'caller_file': 'ui_tests',
@@ -118,7 +83,18 @@
def setUp(self):
super().setUp()
- patch()
+
+ pywikibot.bot.set_interface('terminal')
+
+ self.org_print = pywikibot.bot.ui._print
+ self.org_input = pywikibot.bot.ui._raw_input
+
+ pywikibot.bot.ui._print = self._patched_print
+ pywikibot.bot.ui._raw_input = self._patched_input
+
+ strout.reset()
+ strerr.reset()
+ strin.reset()
pywikibot.config.colorized_output = True
pywikibot.config.transliterate = False
@@ -127,7 +103,24 @@
def tearDown(self):
super().tearDown()
- unpatch()
+
+ pywikibot.bot.ui._print = self.org_print
+ pywikibot.bot.ui._raw_input = self.org_input
+
+ pywikibot.bot.set_interface('buffer')
+
+ def _patched_print(self, text, target_stream):
+ try:
+ stream = patched_streams[target_stream]
+ except KeyError:
+ expected = pywikibot.userinterfaces.win32_unicode.UnicodeOutput
+ assert isinstance(target_stream, expected)
+ assert target_stream._stream
+ stream = patched_streams[target_stream._stream]
+ self.org_print(text, stream)
+
+ def _patched_input(self):
+ return strin._stream.readline().strip()
class TestTerminalOutput(UITestCase):
@@ -152,7 +145,11 @@
logger.log(level, text, extra=loggingcontext)
self.assertEqual(newstdout.getvalue(), out)
self.assertEqual(newstderr.getvalue(), err)
- patch() # reset terminal files
+
+ # reset terminal files
+ strout.reset()
+ strerr.reset()
+ strin.reset()
def test_output(self):
pywikibot.output('output')
@@ -360,7 +357,7 @@
"""Terminal output transliteration tests."""
def testOutputTransliteratedUnicodeText(self):
- pywikibot.ui.encoding = 'latin-1'
+ pywikibot.bot.ui.encoding = 'latin-1'
pywikibot.config.transliterate = True
pywikibot.output('abcd АБГД αβγδ あいうえお')
self.assertEqual(newstdout.getvalue(), '')
@@ -544,8 +541,5 @@
if __name__ == '__main__': # pragma: no cover
- try:
- with suppress(SystemExit):
- unittest.main()
- finally:
- unpatch()
+ with suppress(SystemExit):
+ unittest.main()
To view, visit change 678644. To unsubscribe, or for help writing mail filters, visit settings.