Pywikibot-commits

pywikibot-commits@lists.wikimedia.org

13500 discussions

[Gerrit] ...core[master]: [IMPR] retrieved watchlist in parallel tasks.
by jenkins-bot (Code Review) 10 Sep '22

10 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/826879 ) Change subject: [IMPR] retrieved watchlist in parallel tasks. ...................................................................... [IMPR] retrieved watchlist in parallel tasks. - retrieved in parallel tasks to load data within few seconds. This makes all requests about eight times faster. - reuse count_watchlist_all with refresh_new() function - add processing time counter Change-Id: I70ff846ad5a144e8e1cbbaa939f8fd0e8430c27d --- M scripts/watchlist.py 1 file changed, 38 insertions(+), 27 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/watchlist.py b/scripts/watchlist.py index d5135d9..fefb011 100755 --- a/scripts/watchlist.py +++ b/scripts/watchlist.py @@ -18,13 +18,18 @@ that the bot is connected to. -new Load watchlists for all wikis where accounts is setting in user-config.py + +.. versionchanged:: 7.7 + watchlist is retrieved in parallel tasks. """ # # (C) Pywikibot team, 2005-2022 # # Distributed under the terms of the MIT license. # +import datetime import os +from concurrent.futures import ThreadPoolExecutor, as_completed import pywikibot from pywikibot import config @@ -50,16 +55,20 @@ .format(watchlist_count)) -def count_watchlist_all() -> None: +def count_watchlist_all(quiet=False) -> None: """Count only the total number of page(s) in watchlist for all wikis.""" - wl_count_all = 0 - pywikibot.output('Counting pages in watchlists of all wikis...') - for family in config.usernames: - for lang in config.usernames[family]: - site = pywikibot.Site(lang, family) - wl_count_all += len(refresh(site)) - pywikibot.output('There are a total of {} page(s) in the watchlists' - 'for all wikis.'.format(wl_count_all)) + if not quiet: + pywikibot.info('Counting pages in watchlists of all wikis...') + + with ThreadPoolExecutor() as executor: + futures = {executor.submit(refresh, pywikibot.Site(lang, family)) + for family in config.usernames + for lang in config.usernames[family]} + wl_count_all = sum(len(future.result()) + for future in as_completed(futures)) + if not quiet: + pywikibot.info('There are a total of {} page(s) in the watchlists for ' + 'all wikis.'.format(wl_count_all)) def isWatched(pageName, site=None): # noqa: N802, N803 @@ -79,31 +88,30 @@ cache_path = CachedRequest._get_cache_dir() files = os.scandir(cache_path) seen = set() - for filename in files: - entry = CacheEntry(cache_path, filename) - entry._load_cache() - entry.parse_key() - entry._rebuild() - if entry.site in seen: - continue + with ThreadPoolExecutor() as executor: + for filename in files: + entry = CacheEntry(cache_path, filename) + entry._load_cache() + entry.parse_key() + entry._rebuild() + if entry.site in seen: + continue - # for generator API usage we have to check the modules - modules = entry._params.get('modules', []) - modules_found = any(mod.endswith('watchlistraw') for mod in modules) - # for list API usage 'watchlistraw' is directly found - if modules_found or 'watchlistraw' in entry._data: - refresh(entry.site) - seen.add(entry.site) + # for generator API usage we have to check the modules + modules = entry._params.get('modules', []) + modules_found = any(module.endswith('watchlistraw') + for module in modules) + # for list API usage 'watchlistraw' is directly found + if modules_found or 'watchlistraw' in entry._data: + executor.submit(refresh, entry.site) + seen.add(entry.site) def refresh_new() -> None: """Load watchlists of all wikis for accounts set in user-config.py.""" pywikibot.output( 'Downloading all watchlists for your accounts in user-config.py') - for family in config.usernames: - for lang in config.usernames[family]: - site = pywikibot.Site(lang, family) - refresh(site) + count_watchlist_all(quiet=True) def main(*args: str) -> None: @@ -147,4 +155,7 @@ if __name__ == '__main__': + start = datetime.datetime.now() main() + pywikibot.info('\nExecution time: {} seconds' + .format((datetime.datetime.now() - start).seconds)) -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/826879 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I70ff846ad5a144e8e1cbbaa939f8fd0e8430c27d Gerrit-Change-Number: 826879 Gerrit-PatchSet: 4 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [tests] Display 1 is running on github action
by Xqt (Code Review) 09 Sep '22

09 Sep '22

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831124 ) Change subject: [tests] Display 1 is running on github action ...................................................................... [tests] Display 1 is running on github action Change-Id: I35dbe0f2f236938db41b245de59f5157a0a14e76 --- M tests/tk_tests.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/tests/tk_tests.py b/tests/tk_tests.py index 430f4e3..47a1071 100755 --- a/tests/tk_tests.py +++ b/tests/tk_tests.py @@ -24,7 +24,7 @@ """Set virtual display environment.""" super().setUpClass() cls.env = os.environ.get('DISPLAY') - os.environ['DISPLAY'] = ':0.0' + os.environ['DISPLAY'] = ':1.0' @classmethod def tearDownClass(cls): -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831124 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I35dbe0f2f236938db41b245de59f5157a0a14e76 Gerrit-Change-Number: 831124 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [tests] enable automatic tk_tests
by jenkins-bot (Code Review) 09 Sep '22

09 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831105 ) Change subject: [tests] enable automatic tk_tests ...................................................................... [tests] enable automatic tk_tests - pillow is needed to run tests, require it with test module - get the show_dialog result and add some tests in testTkdialog - use event loop and run skip_file() after few microseconds to press "skip" - use event loop and run pressedOK() after few microseconds to press "Ok" in TestTkinter and test text content - decrease tk size because 1600/1000 looks too big Change-Id: I2e1b3e2d7029fb311ec3ccc3bc7cf93ff5cd9cd0 --- M pywikibot/config.py M tests/tk_tests.py 2 files changed, 18 insertions(+), 11 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/config.py b/pywikibot/config.py index 5a3626a..7207fec 100644 --- a/pywikibot/config.py +++ b/pywikibot/config.py @@ -538,8 +538,8 @@ # An indication of the size of your screen, or rather the size of the screen # to be shown, for flickrripper -tkhorsize = 1600 -tkvertsize = 1000 +tkhorsize = 1280 +tkvertsize = 800 # ############# EXTERNAL EDITOR SETTINGS ############## # The command for the editor you want to use. If set to None, a simple Tkinter diff --git a/tests/tk_tests.py b/tests/tk_tests.py index f94dffd..430f4e3 100755 --- a/tests/tk_tests.py +++ b/tests/tk_tests.py @@ -42,11 +42,16 @@ def testTkdialog(self): """Test Tk dialog.""" - try: - box = Tkdialog('foo', 'tests/data/MP_sounds.png', 'MP_sounds.png') - box.show_dialog() - except ImportError as e: - pywikibot.warning(e) + desc = 'foo' + image = 'tests/data/images/MP_sounds.png' + filename = image.rsplit('/', 1)[1] + box = Tkdialog(desc, image, filename) + # skip after ~100 ms + box.root.after(100, lambda: box.skip_file()) + description, name, skip = box.show_dialog() + self.assertEqual(description, desc) + self.assertEqual(name, filename) + self.assertTrue(skip) class TestTkinter(TkinterTestsBase, DefaultSiteTestCase): @@ -58,15 +63,17 @@ root = tkinter.Tk() root.resizable(width=tkinter.FALSE, height=tkinter.FALSE) root.title('pywikibot GUI') - page = pywikibot.Page(pywikibot.Site(), 'Main Page') + page = pywikibot.Page(self.site, 'Main Page') content = page.get() myapp = EditBoxWindow(root) - myapp.bind('<Control-d>', myapp.debug) - v = myapp.edit(content, highlight=page.title()) - self.assertIsNone(v) + root.after(100, lambda: myapp.pressedOK()) + text = myapp.edit(content, highlight=page.title()) + self.assertIsNotNone(text) + self.assertIn('Main Page', text) @require_modules('tkinter') +@require_modules('PIL') def setUpModule(): """Skip tests if tkinter is not installed. Otherwise import it.""" global EditBoxWindow, Tkdialog, tkinter -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831105 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I2e1b3e2d7029fb311ec3ccc3bc7cf93ff5cd9cd0 Gerrit-Change-Number: 831105 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [tests] create DISPLAY environment variable to run tkinter tests
by jenkins-bot (Code Review) 09 Sep '22

09 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831091 ) Change subject: [tests] create DISPLAY environment variable to run tkinter tests ...................................................................... [tests] create DISPLAY environment variable to run tkinter tests Change-Id: I912d9a5ad72231033c1fb6fddcfd773b359ec74f --- M tests/tk_tests.py 1 file changed, 25 insertions(+), 5 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/tk_tests.py b/tests/tk_tests.py index 58a283d..f94dffd 100755 --- a/tests/tk_tests.py +++ b/tests/tk_tests.py @@ -5,6 +5,7 @@ # # Distributed under the terms of the MIT license. # +import os import unittest from contextlib import suppress @@ -12,12 +13,33 @@ from tests.aspects import DefaultSiteTestCase, TestCase, require_modules -class TestTkdialog(TestCase): +class TkinterTestsBase(TestCase): - """Test Tkdialog.""" + """TestCase base for Tkinter tests.""" net = True + @classmethod + def setUpClass(cls): + """Set virtual display environment.""" + super().setUpClass() + cls.env = os.environ.get('DISPLAY') + os.environ['DISPLAY'] = ':0.0' + + @classmethod + def tearDownClass(cls): + """Restore the display environment value.""" + if not cls.env: + del os.environ['DISPLAY'] + else: + os.environ['DISPLAY'] = cls.env + super().tearDownClass() + + +class TestTkdialog(TkinterTestsBase): + + """Test Tkdialog.""" + def testTkdialog(self): """Test Tk dialog.""" try: @@ -27,12 +49,10 @@ pywikibot.warning(e) -class TestTkinter(DefaultSiteTestCase): +class TestTkinter(TkinterTestsBase, DefaultSiteTestCase): """Test Tkinter.""" - net = True - def testTkinter(self): """Test Tkinter window.""" root = tkinter.Tk() -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831091 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I912d9a5ad72231033c1fb6fddcfd773b359ec74f Gerrit-Change-Number: 831091 Gerrit-PatchSet: 3 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [tests] Remove PYWIKIBOT_TEST_GUI variable
by jenkins-bot (Code Review) 09 Sep '22

09 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831065 ) Change subject: [tests] Remove PYWIKIBOT_TEST_GUI variable ...................................................................... [tests] Remove PYWIKIBOT_TEST_GUI variable Remove PYWIKIBOT_TEST_GUI variable which isn't used by any test. Skip tests if tkinter module is not installed. Change-Id: I092214361bda86f5adb6fb425a6486f1c151b36e --- M tests/tk_tests.py 1 file changed, 6 insertions(+), 11 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/tk_tests.py b/tests/tk_tests.py index a746cee..58a283d 100755 --- a/tests/tk_tests.py +++ b/tests/tk_tests.py @@ -5,18 +5,11 @@ # # Distributed under the terms of the MIT license. # -import os import unittest from contextlib import suppress import pywikibot -from tests.aspects import DefaultSiteTestCase, TestCase - - -if os.environ.get('PYWIKIBOT_TEST_GUI', '0') == '1': - import tkinter - - from pywikibot.userinterfaces.gui import EditBoxWindow, Tkdialog +from tests.aspects import DefaultSiteTestCase, TestCase, require_modules class TestTkdialog(TestCase): @@ -53,10 +46,12 @@ self.assertIsNone(v) +@require_modules('tkinter') def setUpModule(): - """Skip Travis tests if PYWIKIBOT_TEST_GUI variable is not set.""" - if os.environ.get('PYWIKIBOT_TEST_GUI', '0') != '1': - raise unittest.SkipTest('Tkinter tests are disabled on Travis-CI') + """Skip tests if tkinter is not installed. Otherwise import it.""" + global EditBoxWindow, Tkdialog, tkinter + import tkinter + from pywikibot.userinterfaces.gui import EditBoxWindow, Tkdialog if __name__ == '__main__': # pragma: no cover -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831065 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I092214361bda86f5adb6fb425a6486f1c151b36e Gerrit-Change-Number: 831065 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [doc] Add description for PYWIKIBOT_TEST_AUTORUN and PYWIKIBOT_TEST_M...
by jenkins-bot (Code Review) 09 Sep '22

09 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831060 ) Change subject: [doc] Add description for PYWIKIBOT_TEST_AUTORUN and PYWIKIBOT_TEST_MODULES ...................................................................... [doc] Add description for PYWIKIBOT_TEST_AUTORUN and PYWIKIBOT_TEST_MODULES Change-Id: Iae2b53210eebc3183f578603649cfe20232f7a9b --- M tests/README.rst M tests/script_tests.py 2 files changed, 20 insertions(+), 1 deletion(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/README.rst b/tests/README.rst index f53fdd6..cbaee97 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -105,11 +105,29 @@ Environment variables ===================== +**PYWIKIBOT_TEST_AUTORUN** + Enable script tests running automatically without specifying a generator. + The scripts are listed in :attr:`tests.script_tests.auto_run_script_list`. + Currently only *deeptest* uses it. + +**PYWIKIBOT_TEST_MODULES** + Only run tests given with this environment variable. Multiple tests must be + separated by a ``,`` without any white space. Available library tests are + listed in :ref:`Library tests` and script tests can be found in + :ref:`Script tests`. To enable only :mod:`tests.site_tests` and + :mod:`tests.wikibase_tests` set the environment variable as:: + + PYWIKIBOT_TEST_MODULES=site,wikibase + + .. note:: test names must be given without subsequent ``_tests``. + **PYWIKIBOT_TEST_RUNNING** This environment variable skips tests instead of raising :exc:`exceptions.MaxlagTimeoutError` when maximum retries attempted due to maxlag without success. It is also used by :mod:`tests.script_tests` for code - coverage. Github actions and Appveyor tests activate this variable. + coverage. Github actions and Appveyor tests activate this variable:: + + PYWIKIBOT_TEST_RUNNING=1 **PYWIKIBOT_TEST_WRITE** There are also several other 'write' tests which also attempt to perform diff --git a/tests/script_tests.py b/tests/script_tests.py index 7afeb2f..777815b 100755 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -76,6 +76,7 @@ '8/80/Wikipedia-logo-v2.svg\n\n\n', } +#: auto_run_script_list = [ 'archivebot', 'blockpageschecker', -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831060 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Iae2b53210eebc3183f578603649cfe20232f7a9b Gerrit-Change-Number: 831060 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [tests] Rename PYWIKIBOT_TESTS_RUNNING environment variable
by jenkins-bot (Code Review) 09 Sep '22

09 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831052 ) Change subject: [tests] Rename PYWIKIBOT_TESTS_RUNNING environment variable ...................................................................... [tests] Rename PYWIKIBOT_TESTS_RUNNING environment variable - rename PYWIKIBOT_TESTS_RUNNING to PYWIKIBOT_TEST_RUNNING to be consistent with other PYWIKIBOT_TEST_... environment variables - add this environment variable to tests/README.rst and documentation Change-Id: I8d5b4d454ae9662311993898567dddb2e3d17e8b --- M .appveyor.yml M .github/workflows/login_tests-ci.yml M .github/workflows/pywikibot-ci.yml M pywikibot/data/api/_requests.py M tests/README.rst M tests/script_tests.py 6 files changed, 32 insertions(+), 22 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/.appveyor.yml b/.appveyor.yml index 75a438e..f534951 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,7 +9,7 @@ PYWIKIBOT_PASSWD_FILE: "%appdata%\\Pywikibot\\passwordfile" PYSETUP_TEST_EXTRAS: "1" - PYWIKIBOT_TESTS_RUNNING: "1" + PYWIKIBOT_TEST_RUNNING: "1" matrix: diff --git a/.github/workflows/login_tests-ci.yml b/.github/workflows/login_tests-ci.yml index c6a97f2..55d5246 100644 --- a/.github/workflows/login_tests-ci.yml +++ b/.github/workflows/login_tests-ci.yml @@ -12,7 +12,7 @@ - completed env: - PYWIKIBOT_TESTS_RUNNING: 1 + PYWIKIBOT_TEST_RUNNING: 1 PYWIKIBOT_LOGIN_LOGOUT: 1 PYWIKIBOT_USERNAME: Pywikibot-test diff --git a/.github/workflows/pywikibot-ci.yml b/.github/workflows/pywikibot-ci.yml index 21be465..ae29edc 100644 --- a/.github/workflows/pywikibot-ci.yml +++ b/.github/workflows/pywikibot-ci.yml @@ -11,7 +11,7 @@ branches: [ master ] env: - PYWIKIBOT_TESTS_RUNNING: 1 + PYWIKIBOT_TEST_RUNNING: 1 PYWIKIBOT_USERNAME: Pywikibot-test jobs: diff --git a/pywikibot/data/api/_requests.py b/pywikibot/data/api/_requests.py index 4d26391..5e1d564 100644 --- a/pywikibot/data/api/_requests.py +++ b/pywikibot/data/api/_requests.py @@ -1110,7 +1110,7 @@ raise RuntimeError(result) msg = 'Maximum retries attempted due to maxlag without success.' - if os.environ.get('PYWIKIBOT_TESTS_RUNNING', '0') == '1': + if os.environ.get('PYWIKIBOT_TEST_RUNNING', '0') == '1': import unittest raise unittest.SkipTest(msg) diff --git a/tests/README.rst b/tests/README.rst index 85d1ea5..f53fdd6 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -105,29 +105,39 @@ Environment variables ===================== -PYWIKIBOT_TEST_WRITE_FAIL - There are a set of 'edit failure' tests, which attempt to write to the wikis - and **should** fail. If there is a bug in pywikibot or MediaWiki, these - tests **may** actually perform a write operation. +**PYWIKIBOT_TEST_RUNNING** + This environment variable skips tests instead of raising + :exc:`exceptions.MaxlagTimeoutError` when maximum retries attempted due to + maxlag without success. It is also used by :mod:`tests.script_tests` for code + coverage. Github actions and Appveyor tests activate this variable. - These 'edit failure' tests are disabled by default. On Travis they are enabled - by default on builds by any other GitHub account except 'wikimedia'. +**PYWIKIBOT_TEST_WRITE** + There are also several other 'write' tests which also attempt to perform + write operations successfully. These **will** write to the wikis, and they + should always only write to 'test' wikis. - To disable 'edit failure' tests, set PYWIKIBOT_TEST_WRITE_FAIL=0 + These 'write' tests are disabled by default, and currently cannot be + run on Travis or AppVeyor as they require interaction using a terminal. Also + enabling them won't enable 'edit failure' tests. -PYWIKIBOT_TEST_WRITE - There are also several other 'write' tests which also attempt to perform - write operations successfully. These **will** write to the wikis, and they - should always only write to 'test' wikis. + To enable 'write' tests, set:: - These 'write' tests are disabled by default, and currently cannot be - run on Travis or AppVeyor as they require interaction using a terminal. Also - enabling them won't enable 'edit failure' tests. + PYWIKIBOT_TEST_WRITE=1 - To enable 'write' tests, set PYWIKIBOT_TEST_WRITE=1 +**PYWIKIBOT_TEST_WRITE_FAIL** + There are a set of 'edit failure' tests, which attempt to write to the wikis + and **should** fail. If there is a bug in pywikibot or MediaWiki, these + tests **may** actually perform a write operation. -Enabling only 'edit failure' tests or 'write' tests won't enable the other tests -automatically. + These 'edit failure' tests are disabled by default. On Travis they are enabled + by default on builds by any other GitHub account except 'wikimedia'. + + To disable 'edit failure' tests, set:: + + PYWIKIBOT_TEST_WRITE_FAIL=0 + +.. note:: Enabling only 'edit failure' tests or 'write' tests won't enable the other tests + automatically. Decorators ========== diff --git a/tests/script_tests.py b/tests/script_tests.py index d499269..7afeb2f 100755 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -17,7 +17,7 @@ from tests.utils import execute_pwb -ci_test_run = os.environ.get('PYWIKIBOT_TESTS_RUNNING', '0') == '1' +ci_test_run = os.environ.get('PYWIKIBOT_TEST_RUNNING', '0') == '1' scripts_path = join_root_path('scripts') # login scritpt should be the first to test -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831052 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I8d5b4d454ae9662311993898567dddb2e3d17e8b Gerrit-Change-Number: 831052 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [doc] Update test documentation
by Xqt (Code Review) 09 Sep '22

09 Sep '22

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831043 ) Change subject: [doc] Update test documentation ...................................................................... [doc] Update test documentation Add definition list to README.rst for test environment variables Change-Id: I5d402eb4416658b330ef1f1451a0d3b4622dbf22 --- M tests/README.rst 1 file changed, 24 insertions(+), 22 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/tests/README.rst b/tests/README.rst index 9e5e5ee..85d1ea5 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -105,36 +105,38 @@ Environment variables ===================== -There are a set of 'edit failure' tests, which attempt to write to the wikis -and **should** fail. If there is a bug in pywikibot or MediaWiki, these -tests **may** actually perform a write operation. +PYWIKIBOT_TEST_WRITE_FAIL + There are a set of 'edit failure' tests, which attempt to write to the wikis + and **should** fail. If there is a bug in pywikibot or MediaWiki, these + tests **may** actually perform a write operation. -These 'edit failure' tests are disabled by default. On Travis they are enabled -by default on builds by any other GitHub account except 'wikimedia'. + These 'edit failure' tests are disabled by default. On Travis they are enabled + by default on builds by any other GitHub account except 'wikimedia'. -To disable 'edit failure' tests, set PYWIKIBOT_TEST_WRITE_FAIL=0 + To disable 'edit failure' tests, set PYWIKIBOT_TEST_WRITE_FAIL=0 -There are also several other 'write' tests which also attempt to perform -write operations successfully. These **will** write to the wikis, and they -should always only write to 'test' wikis. +PYWIKIBOT_TEST_WRITE + There are also several other 'write' tests which also attempt to perform + write operations successfully. These **will** write to the wikis, and they + should always only write to 'test' wikis. -These 'write' tests are disabled by default, and currently cannot be -run on Travis or AppVeyor as they require interaction using a terminal. Also -enabling them won't enable 'edit failure' tests. + These 'write' tests are disabled by default, and currently cannot be + run on Travis or AppVeyor as they require interaction using a terminal. Also + enabling them won't enable 'edit failure' tests. -To enable 'write' tests, set PYWIKIBOT_TEST_WRITE=1 + To enable 'write' tests, set PYWIKIBOT_TEST_WRITE=1 Enabling only 'edit failure' tests or 'write' tests won't enable the other tests automatically. Decorators -===================== +========== pywikibot's test suite, including Python's unittest module, provides decorators to modify the behaviour of the test cases. @unittest.skipIf ------------------ +---------------- Skip a test if the condition is true. Refer to unittest's documentation. :: @@ -145,7 +147,7 @@ def test_skipIf(self): @unittest.skipUnless ---------------------- +-------------------- Skip a test unless the condition is true. Refer to unittest's documentation. :: @@ -246,9 +248,9 @@ Other class attributes ---------------------- -- ``net = False`` : test class does not use a site -- ``dry = True`` : test class can use a fake site object -- ``cached = True``: test class may aggressively cache API responses -- ``login = True`` : test class needs to login to site -- ``rights = '<rights>'`` : test class needs specific rights. Multiple rights must be delimited with `,`. -- ``write = True`` : test class needs to write to a site +- ``net = False``: test class does not use a site +- ``dry = True``: test class can use a fake site object +- ``cached = True``: test class may aggressively cache API responses +- ``login = True``: test class needs to login to site +- ``rights = '<rights>'``: test class needs specific rights. Multiple rights must be delimited with ``,``. +- ``write = True``: test class needs to write to a site -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/831043 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I5d402eb4416658b330ef1f1451a0d3b4622dbf22 Gerrit-Change-Number: 831043 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [IMPR] catch ServerError as a whole in reflinks.py
by Xqt (Code Review) 07 Sep '22

07 Sep '22

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/830617 ) Change subject: [IMPR] catch ServerError as a whole in reflinks.py ...................................................................... [IMPR] catch ServerError as a whole in reflinks.py All server errors derive from ServerError which can be used to catch them all. https://doc.wikimedia.org/pywikibot/master/api_ref/exceptions.html?highligh… Change-Id: Id355cc666813b090924060815507a896684f6821 --- M scripts/reflinks.py 1 file changed, 2 insertions(+), 8 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/scripts/reflinks.py b/scripts/reflinks.py index 51c0951..7cfba8e 100755 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -64,11 +64,7 @@ from pywikibot.backports import removeprefix from pywikibot.bot import ConfigParserBot, ExistingPageBot, SingleSiteBot from pywikibot.comms.http import get_charset_from_content_type -from pywikibot.exceptions import ( - FatalServerError, - Server414Error, - Server504Error, -) +from pywikibot.exceptions import ServerError from pywikibot.pagegenerators import ( XMLDumpPageGenerator as _XMLDumpPageGenerator, ) @@ -638,9 +634,7 @@ except (ValueError, # urllib3.LocationParseError derives from it OSError, httplib.error, - FatalServerError, - Server414Error, - Server504Error) as e: + ServerError) as e: pywikibot.output( "{err.__class__.__name__}: Can't retrieve url {url}: {err}" .format(url=ref.url, err=e)) -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/830617 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Id355cc666813b090924060815507a896684f6821 Gerrit-Change-Number: 830617 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [IMPR] Some improvements
by jenkins-bot (Code Review) 07 Sep '22

07 Sep '22

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/830572 ) Change subject: [IMPR] Some improvements ...................................................................... [IMPR] Some improvements - allow defusedxml to be used with xmlreader.py - simplify assignments in dataextend.py - always return a list within Analyzer.findclaims() - validate server certificate's hostname in dataextend.py - update ignore-names for pep8-naming to its default Change-Id: I6d9c469450d64ba5e32ca696d1f3790d417adbdb --- M pywikibot/xmlreader.py M scripts/dataextend.py M tox.ini 3 files changed, 27 insertions(+), 19 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/xmlreader.py b/pywikibot/xmlreader.py index 8277b97..69712c1 100644 --- a/pywikibot/xmlreader.py +++ b/pywikibot/xmlreader.py @@ -1,11 +1,14 @@ -""" -XML reading module. +"""XML reading module. Each XmlEntry object represents a page, as read from an XML source The XmlDump class reads a pages_current XML dump (like the ones offered on https://dumps.wikimedia.org/backup-index.html) and offers a generator over XmlEntry objects which can be used by other bots. + +.. versionchanged:: 7.7 + *defusedxml* is used in favour of *xml.etree* if present to prevent + vulnerable XML attacks. *defusedxml* 0.7.1 or higher is recommended. """ # # (C) Pywikibot team, 2005-2022 @@ -14,7 +17,11 @@ # import re from typing import Optional -from xml.etree.ElementTree import iterparse, ParseError + +try: + from defusedxml.ElementTree import iterparse, ParseError +except ImportError: + from xml.etree.ElementTree import iterparse, ParseError from pywikibot.backports import Callable, Type from pywikibot.tools import open_archive diff --git a/scripts/dataextend.py b/scripts/dataextend.py index c234485..ac6b861 100644 --- a/scripts/dataextend.py +++ b/scripts/dataextend.py @@ -67,13 +67,13 @@ from contextlib import suppress from html import unescape from textwrap import shorten -from typing import Tuple +from typing import Optional from urllib.error import HTTPError, URLError from urllib.parse import quote, unquote from urllib.request import urlopen import pywikibot -from pywikibot.backports import List +from pywikibot.backports import List, Tuple from pywikibot.bot import input_yn, SingleSiteBot, suggest_help from pywikibot.data import sparql from pywikibot.exceptions import ( @@ -1242,7 +1242,7 @@ term = term.split('(')[0] if ',' in term: if term.split(',')[1].strip().lower() in ['jr', 'sr']: - term = term + '.' + term += '.' else: if term.strip()[-1] != term.strip()[-1].lower(): term = term.strip() + '.' @@ -1283,14 +1283,13 @@ answer = None return answer - def findclaims(self): - if not self.id: - return + def findclaims(self) -> List[Tuple[str, str, Optional['Analyzer']]]: + if not self.id or not (self.url or self.sparqlquery): + return [] + self.html = '' - if not self.url and not self.sparqlquery: - return newclaims = [] - pywikibot.output() + pywikibot.info() pagerequest = None if not self.skipfirst: try: @@ -1316,7 +1315,6 @@ pywikibot.output('Unable to receive page {} - not unicode?' .format(self.url)) pagerequest = None - self.html = '' if pagerequest: pagebytes = pagerequest.read() @@ -1330,6 +1328,8 @@ pywikibot.output('Getting {}'.format(extraurl)) if 'https' in self.url: context = ssl._create_unverified_context() + # validate server certificate's hostname is recommened + context.check_hostname = True pagerequest = urlopen(extraurl, context=context) else: pagerequest = urlopen(extraurl) @@ -1338,14 +1338,15 @@ else: pagebytes = pagerequest.read() try: - self.html = self.html + '\n' + pagebytes.decode('utf-8') + self.html += '\n' + pagebytes.decode('utf-8') except UnicodeDecodeError: - self.html = self.html + '\n' + str(pagebytes) + self.html += '\n' + str(pagebytes) if self.sparqlquery: self.html = str(sparql.SparqlQuery().select(self.sparqlquery)) + if not self.html: - return + return [] if self.escapeunicode: self.html = self.html.encode().decode('unicode-escape') @@ -5722,11 +5723,11 @@ section = self.findbyre( r'(?s)<div class="detail_label">Artistic Role\(s\):</div>\s*<div class="detail_text">(.*?)<', html) if section: - result = result + self.findallbyre(r'([^,]*)', section, 'occupation') + result += self.findallbyre(r'([^,]*)', section, 'occupation') section = self.findbyre( r'(?s)<div class="detail_label">Other Occupation\(s\):</div>\s*<div class="detail_text">(.*?)<', html) if section: - result = result + self.findallbyre(r'([^,]*)', section, 'occupation') + result += self.findallbyre(r'([^,]*)', section, 'occupation') return result def findresidences(self, html: str): diff --git a/tox.ini b/tox.ini index 3b35408..654e9aa 100644 --- a/tox.ini +++ b/tox.ini @@ -219,7 +219,7 @@ # pep8-naming classmethod-decorators = classmethod,classproperty # required with pep8-naming < 0.13 -ignore-names = setUp,tearDown,setUpClass,tearDownClass,setUpModule,tearDownModule,maxDiff +ignore-names = setUp,tearDown,setUpClass,tearDownClass,setUpModule,tearDownModule,asyncSetUp,asyncTearDown,setUpTestData,failureException,longMessage,maxDiff [isort] atomic = true -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/830572 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I6d9c469450d64ba5e32ca696d1f3790d417adbdb Gerrit-Change-Number: 830572 Gerrit-PatchSet: 6 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits