Pywikibot-commits

pywikibot-commits@lists.wikimedia.org

1 participants
13475 discussions

[Gerrit] ...core[master]: [tests] Simplify TestCaseBase._get_gen_pages() method
by jenkins-bot (Code Review) 26 Jun '24

26 Jun '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049916?usp=email ) Change subject: [tests] Simplify TestCaseBase._get_gen_pages() method ...................................................................... [tests] Simplify TestCaseBase._get_gen_pages() method - remove count parameter of TestCaseBase._get_gen_pages() because it was never used within the last 9 years - update the calling methods accordingly Change-Id: I3bb43a1eb07784ea525511cafdd1dfbb30ebaa2d --- M tests/aspects.py 1 file changed, 27 insertions(+), 42 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/tests/aspects.py b/tests/aspects.py index 18da72f..3b22a58 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -12,7 +12,6 @@ from __future__ import annotations import inspect -import itertools import os import re import sys @@ -27,7 +26,7 @@ import pywikibot from pywikibot import Site, config -from pywikibot.backports import removeprefix, removesuffix +from pywikibot.backports import Iterable, Iterator, removeprefix, removesuffix from pywikibot.comms import http from pywikibot.data.api import Request as _original_Request from pywikibot.exceptions import ( @@ -133,33 +132,21 @@ self.assertIn(page.namespace(), namespaces, f'{page} not in namespace {namespaces!r}') - def _get_gen_pages(self, gen, count: int = None, site=None): - """ - Get pages from gen, asserting they are Page from site. + def _get_gen_pages(self, + gen: Iterable[pywikibot.Page], + site: pywikibot.site.APISite = None) -> None: + """Get pages from gen, asserting they are Page from site. - Iterates at most two greater than count, including the - Page after count if it exists, and then a Page with title '...' - if additional items are in the iterator. + .. versionchanged:: 9.3 + the *count* parameter was dropped; all pages from *gen* are + tested. :param gen: Page generator - :type gen: typing.Iterable[pywikibot.Page] - :param count: number of pages to get :param site: Site of expected pages - :type site: pywikibot.site.APISite + :meta public: """ - original_iter = iter(gen) - - gen = itertools.islice(original_iter, 0, count) - gen_pages = list(gen) - with suppress(StopIteration): - gen_pages.append(next(original_iter)) - next(original_iter) - if not site: - site = gen_pages[0].site - gen_pages.append(pywikibot.Page(site, '...')) - for page in gen_pages: self.assertIsInstance(page, pywikibot.Page) if site: @@ -167,9 +154,9 @@ return gen_pages - def _get_gen_titles(self, gen, count: int, site=None) -> list[str]: + def _get_gen_titles(self, gen, site=None) -> list[str]: """Return a list of page titles of given iterable.""" - return [page.title() for page in self._get_gen_pages(gen, count, site)] + return [page.title() for page in self._get_gen_pages(gen, site)] @staticmethod def _get_canonical_titles(titles, site=None): @@ -220,38 +207,36 @@ self.assertEqual(page_namespaces, namespaces) - def assertPageTitlesEqual(self, gen, titles, site=None): - """ - Test that pages in gen match expected titles. - - Only iterates to the length of titles plus two. + def assertPageTitlesEqual( + self, + gen: Iterable[pywikibot.Page], + titles: Iterator[str], + site: pywikibot.site.APISite | None = None + ) -> None: + """Test that pages in gen match expected titles. :param gen: Page generator - :type gen: typing.Iterable[pywikibot.Page] :param titles: Expected titles - :type titles: iterator :param site: Site of expected pages - :type site: pywikibot.site.APISite """ titles = self._get_canonical_titles(titles, site) - gen_titles = self._get_gen_titles(gen, len(titles), site) + gen_titles = self._get_gen_titles(gen, site) self.assertEqual(gen_titles, titles) - def assertPageTitlesCountEqual(self, gen, titles, site=None): - """ - Test that pages in gen match expected titles, regardless of order. - - Only iterates to the length of titles plus two. + def assertPageTitlesCountEqual( + self, + gen: Iterable[pywikibot.Page], + titles: Iterator[str], + site: pywikibot.site.APISite | None = None + ) -> None: + """Test that pages in gen match expected titles, regardless of order. :param gen: Page generator - :type gen: typing.Iterable[pywikibot.Page] :param titles: Expected titles - :type titles: iterator :param site: Site of expected pages - :type site: pywikibot.site.APISite """ titles = self._get_canonical_titles(titles, site) - gen_titles = self._get_gen_titles(gen, len(titles), site) + gen_titles = self._get_gen_titles(gen, site) self.assertCountEqual(gen_titles, titles) def assertAPIError(self, code, info=None, callable_obj=None, *args, -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049916?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I3bb43a1eb07784ea525511cafdd1dfbb30ebaa2d Gerrit-Change-Number: 1049916 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [tests] fix github action config file
by Xqt (Code Review) 24 Jun '24

24 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049256?usp=email ) Change subject: [tests] fix github action config file ...................................................................... [tests] fix github action config file Change-Id: I5c2f06d2d9767acf88a21bdb71167768c261e1c3 Signed-off-by: Xqt <info(a)gno.de> --- M .github/workflows/windows_tests.yml 1 file changed, 3 insertions(+), 3 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/.github/workflows/windows_tests.yml b/.github/workflows/windows_tests.yml index e61495a..afa0b9a 100644 --- a/.github/workflows/windows_tests.yml +++ b/.github/workflows/windows_tests.yml @@ -60,9 +60,9 @@ python -c "import struct; print('PYTHON_ARCH:', struct.calcsize('P') << 3)" python -m pip install --upgrade pip pip --version - if [ ${{matrix.python-version}} == '3.7.0'; then pip install "urllib3<2.0"; fi - if [ ${{matrix.python-version}} == '3.7.0'; then pip install -U setuptools; fi - if [ ${{matrix.python-version}} >= '3.12'; then pip install -U setuptools; fi + if [ ${{matrix.python-version}} == '3.7.0' ]; then pip install "urllib3<2.0"; fi + if [ ${{matrix.python-version}} == '3.7.0' ]; then pip install -U setuptools; fi + if [ ${{matrix.python-version}} >= '3.12' ]; then pip install -U setuptools; fi if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install wikitextparser -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049256?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I5c2f06d2d9767acf88a21bdb71167768c261e1c3 Gerrit-Change-Number: 1049256 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [tests] Run windows tests on windows
by Xqt (Code Review) 24 Jun '24

24 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049233?usp=email ) Change subject: [tests] Run windows tests on windows ...................................................................... [tests] Run windows tests on windows Change-Id: I97fa8b82401c97b2168b2e7bd933677bbec91854 --- M .github/workflows/windows_tests.yml 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: jenkins-bot: Verified Xqt: Verified; Looks good to me, approved diff --git a/.github/workflows/windows_tests.yml b/.github/workflows/windows_tests.yml index 661f476..e61495a 100644 --- a/.github/workflows/windows_tests.yml +++ b/.github/workflows/windows_tests.yml @@ -17,7 +17,7 @@ jobs: build: - runs-on: ${{ matrix.os || 'ubuntu-latest' }} + runs-on: ${{ matrix.os || 'windows-latest' }} continue-on-error: ${{ matrix.experimental || false }} timeout-minutes: 100 -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049233?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I97fa8b82401c97b2168b2e7bd933677bbec91854 Gerrit-Change-Number: 1049233 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [tests] Add experimental Windows tests as successor for Appveyor tests
by Xqt (Code Review) 24 Jun '24

24 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049229?usp=email ) Change subject: [tests] Add experimental Windows tests as successor for Appveyor tests ...................................................................... [tests] Add experimental Windows tests as successor for Appveyor tests Bug: T368192 Change-Id: Ieb737cbebb734cc0babcfab063102d683f70b687 --- M .github/workflows/doctest.yml M .github/workflows/oauth_tests-ci.yml M .github/workflows/pywikibot-ci.yml M .github/workflows/sysop_write_tests-ci.yml A .github/workflows/windows_tests.yml 5 files changed, 131 insertions(+), 3 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/.github/workflows/doctest.yml b/.github/workflows/doctest.yml index 56c5c54..a8c91f0 100644 --- a/.github/workflows/doctest.yml +++ b/.github/workflows/doctest.yml @@ -20,7 +20,7 @@ strategy: fail-fast: false - max-parallel: 14 + max-parallel: 17 matrix: python-version: [ "pypy3.7", "pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha - 3.13.0" ] diff --git a/.github/workflows/oauth_tests-ci.yml b/.github/workflows/oauth_tests-ci.yml index 64598d0..598833d 100644 --- a/.github/workflows/oauth_tests-ci.yml +++ b/.github/workflows/oauth_tests-ci.yml @@ -20,6 +20,7 @@ strategy: fail-fast: false + max-parallel: 8 matrix: python-version: [ "pypy3.7", "pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha - 3.13.0", "3.14-dev" ] diff --git a/.github/workflows/pywikibot-ci.yml b/.github/workflows/pywikibot-ci.yml index d45d1ef..511f07b 100644 --- a/.github/workflows/pywikibot-ci.yml +++ b/.github/workflows/pywikibot-ci.yml @@ -1,5 +1,5 @@ name: Pywikibot CI -# Run Pywikibot tests with a variety of Python versions +# Run Pywikibot tests with a variety of Python versions on: push: @@ -23,7 +23,7 @@ strategy: fail-fast: false - max-parallel: 17 + max-parallel: 18 matrix: python-version: [ "pypy3.7", "pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha - 3.13.0", "3.14-dev" ] diff --git a/.github/workflows/sysop_write_tests-ci.yml b/.github/workflows/sysop_write_tests-ci.yml index 6d692cb..9d6219f 100644 --- a/.github/workflows/sysop_write_tests-ci.yml +++ b/.github/workflows/sysop_write_tests-ci.yml @@ -21,6 +21,7 @@ strategy: fail-fast: false + max-parallel: 2 matrix: python-version: [ "3.7" ] diff --git a/.github/workflows/windows_tests.yml b/.github/workflows/windows_tests.yml new file mode 100644 index 0000000..661f476 --- /dev/null +++ b/.github/workflows/windows_tests.yml @@ -0,0 +1,126 @@ +name: Windows Tests +# Successor for Appveyor Tests + +on: + push: + branches: [ master, stable ] + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYWIKIBOT_TEST_RUNNING: 1 + PYWIKIBOT_USERNAME: Pywikibot-test + +jobs: + build: + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + continue-on-error: ${{ matrix.experimental || false }} + timeout-minutes: 100 + + strategy: + fail-fast: false + max-parallel: 10 + + matrix: + python-version: [ "3.7.0", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha - 3.13.0" ] + python-arch: [ x64, x86 ] + site: [ "wikipedia:en" ] + test_prod_only: [ false ] + experimental: [ true ] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + if: "!endsWith(matrix.python-version, '-dev')" + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.python-arch }} + + # use deadsnakes/action for development releases + - name: Set up development Python ${{ matrix.python-version }} + uses: deadsnakes/action(a)v3.1.0 + if: "endsWith(matrix.python-version, '-dev')" + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.python-arch }} + + - name: Install dependencies + run: | + git submodule update --init + python --version + python -c "import struct; print('PYTHON_ARCH:', struct.calcsize('P') << 3)" + python -m pip install --upgrade pip + pip --version + if [ ${{matrix.python-version}} == '3.7.0'; then pip install "urllib3<2.0"; fi + if [ ${{matrix.python-version}} == '3.7.0'; then pip install -U setuptools; fi + if [ ${{matrix.python-version}} >= '3.12'; then pip install -U setuptools; fi + if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install wikitextparser + + - name: Generate family files + run: | + if [ ${{matrix.family || 0}} == wpbeta ]; then + python pwb.py generate_family_file http://${{matrix.code}}.wikipedia.beta.wmflabs.org/ wpbeta y + fi + if [ ${{matrix.site || 0}} == 'wsbeta:en' ]; then + python pwb.py generate_family_file http://en.wikisource.beta.wmflabs.org/ wsbeta y + fi + + - name: Generate user files + run: | + if [ ${{matrix.site}} != false ]; then + python -Werror::UserWarning -m pwb generate_user_files -site:${{matrix.site}} -user:${{ env.PYWIKIBOT_USERNAME }} -v -debug; + else + python -Werror::UserWarning -m pwb generate_user_files -family:${{matrix.family}} -lang:${{matrix.code}} -user:${{ env.PYWIKIBOT_USERNAME }} -v -debug; + fi + echo "usernames['wikipedia']['en'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "usernames['wikisource']['zh'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "usernames['wikipedia']['test'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "usernames['wikidata']['test'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "usernames['commons']['commons'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "usernames['meta']['meta'] = '${{ env.PYWIKIBOT_USERNAME }}'" >> user-config.py + echo "noisysleep = float('inf')" >> user-config.py + echo "maximum_GET_length = 5000" >> user-config.py + echo "console_encoding = 'utf8'" >> user-config.py + echo "import os" >> user-config.py + echo "password_file = os.path.expanduser('passwordfile')" >> user-config.py + echo "('${{ env.PYWIKIBOT_USERNAME }}', '${{ secrets.PYWIKIBOT_USERPWD }}')" > passwordfile + + - name: Test with unittest or pytest + id: ci_test + continue-on-error: true + timeout-minutes: 90 + env: + PYWIKIBOT_TEST_NO_RC: + ${{ (matrix.site == 'wikisource:zh' || matrix.test_no_rc) && 1 || 0 }} + PYWIKIBOT_TEST_PROD_ONLY: ${{ matrix.test_prod_only && 1 || 0 }} + run: | + python pwb.py version + if [ ${{matrix.site || 0}} != 'wikipedia:test' ]; then + coverage run -m unittest discover -vv -p \"*_tests.py\"; + else + pytest --cov=.; + fi + + - name: Show coverage statistics + run: | + coverage report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Check on failure + if: steps.ci_test.outcome == 'failure' + run: exit 1 -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049229?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Ieb737cbebb734cc0babcfab063102d683f70b687 Gerrit-Change-Number: 1049229 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-CC: jenkins-bot

1 0

[Gerrit] ...core[master]: [make_dist] additional tests for make_dist.py
by Xqt (Code Review) 24 Jun '24

24 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049191?usp=email ) Change subject: [make_dist] additional tests for make_dist.py ...................................................................... [make_dist] additional tests for make_dist.py also update coverage. Change-Id: Ifb12cec13ce6f61f021df3818a2f71b3554e58f0 --- M make_dist.py M tests/make_dist_tests.py 2 files changed, 29 insertions(+), 12 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/make_dist.py b/make_dist.py index cc498e2..2c0ca36 100755 --- a/make_dist.py +++ b/make_dist.py @@ -109,7 +109,7 @@ if self.local or self.remote or self.clear: self.clear_old_dist() if self.clear: - return True # pragma: no cover + return True if self.upgrade: # pragma: no cover check_call('python -m pip install --upgrade pip', shell=True) @@ -125,16 +125,22 @@ for module in ('build', 'twine'): try: import_module(module) - except ModuleNotFoundError as e: # pragma: no cover + except ModuleNotFoundError as e: error(f'<<lightred>>{e}') info('<<lightblue>>You may use -upgrade option to install') return False + return self.build() # pragma: no cover + def build(self) -> bool: # pragma: no cover + """Build the packages. + + .. versionadded:: 9.3 + """ self.copy_files() info('<<lightyellow>>Build package') try: check_call('python -m build') - except Exception as e: # pragma: no cover + except Exception as e: error(e) return False finally: @@ -142,7 +148,7 @@ info('<<lightyellow>>Check package and description') if run('twine check dist/*', shell=True).returncode: - return False # pragma: no cover + return False if self.local: info('<<lightyellow>>Install locally') @@ -153,7 +159,7 @@ if self.remote and input_yn( '<<lightblue>>Upload dist to pypi', automatic_quit=False): - check_call('twine upload dist/*', shell=True) # pragma: no cover + check_call('twine upload dist/*', shell=True) return True @@ -172,7 +178,7 @@ self.target = target self.source = source - def copy_files(self) -> None: + def copy_files(self) -> None: # pragma: no cover """Copy i18n files to pywikibot.scripts folder. Pywikibot i18n files are used for some translations. They are copied @@ -186,7 +192,7 @@ shutil.copytree(self.source, self.target) info('done') - def cleanup(self) -> None: + def cleanup(self) -> None: # pragma: no cover """Remove all copied files from pywikibot scripts folder.""" info('<<lightyellow>>Remove copied files... ', newline=False) shutil.rmtree(self.target) diff --git a/tests/make_dist_tests.py b/tests/make_dist_tests.py index 26d498f..699c666 100755 --- a/tests/make_dist_tests.py +++ b/tests/make_dist_tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Tests for fixes module.""" # -# (C) Pywikibot team, 2022-2023 +# (C) Pywikibot team, 2022-2024 # # Distributed under the terms of the MIT license. # @@ -10,6 +10,7 @@ import sys import unittest +import make_dist from pywikibot import __version__ from tests.aspects import TestCase @@ -22,20 +23,30 @@ def test_handle_args_empty(self): """Test make_dist handle_args function.""" - from make_dist import handle_args - args = handle_args() + args = make_dist.handle_args() self.assertEqual(args, (False, ) * 4) def test_handle_args(self): """Test make_dist handle_args function.""" - from make_dist import handle_args sys.argv += ['-clear', '-local', '-remote', '-upgrade'] - local, remote, clear, upgrade = handle_args() + local, remote, clear, upgrade = make_dist.handle_args() self.assertTrue(local) self.assertEqual(remote, 'dev' not in __version__) self.assertTrue(clear) self.assertTrue(upgrade) + def test_main(self): + """Test main result.""" + saved_argv = sys.argv + sys.argv = saved_argv + ['-clear'] + self.assertTrue(make_dist.main()) + + # no build or twine modules + self.assertFalse(make_dist.main()) + sys.argv = saved_argv + ['-local'] + self.assertFalse(make_dist.main()) + sys.argv = saved_argv + if __name__ == '__main__': unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1049191?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Ifb12cec13ce6f61f021df3818a2f71b3554e58f0 Gerrit-Change-Number: 1049191 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [IMPR] replace pywikibot.bot.output with pywikibot.info
by jenkins-bot (Code Review) 23 Jun '24

23 Jun '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048879?usp=email ) Change subject: [IMPR] replace pywikibot.bot.output with pywikibot.info ...................................................................... [IMPR] replace pywikibot.bot.output with pywikibot.info Change-Id: I517a2fdb0240464829c5ab5992b0880e8559e966 --- M pywikibot/bot.py 1 file changed, 2 insertions(+), 2 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/bot.py b/pywikibot/bot.py index a993a1a..449377a 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -781,8 +781,8 @@ elif option in ('-cosmeticchanges', '-cc'): config.cosmetic_changes = (strtobool(value) if value else not config.cosmetic_changes) - output(f'NOTE: option cosmetic_changes is ' - f'{config.cosmetic_changes}\n') + pywikibot.info(f'NOTE: option cosmetic_changes is ' + f'{config.cosmetic_changes}\n') elif option == '-simulate': config.simulate = value or True # -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048879?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I517a2fdb0240464829c5ab5992b0880e8559e966 Gerrit-Change-Number: 1048879 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [doc] fix spelling mistake
by Xqt (Code Review) 23 Jun '24

23 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048878?usp=email ) Change subject: [doc] fix spelling mistake ...................................................................... [doc] fix spelling mistake Change-Id: I12ac7ef7963e8a8422c2380df793e4a23d841a47 --- M pywikibot/page/_toolforge.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/pywikibot/page/_toolforge.py b/pywikibot/page/_toolforge.py index b701c1f..3342040 100644 --- a/pywikibot/page/_toolforge.py +++ b/pywikibot/page/_toolforge.py @@ -114,7 +114,7 @@ revid: int | None = None, date: DATETYPE = None, ) -> dict[str, tuple[int, float]]: - """Retrieve authorship attributon of an article. + """Retrieve authorship attribution of an article. This method uses XTools/Authorship to retrieve the authors measured by character count. -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048878?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I12ac7ef7963e8a8422c2380df793e4a23d841a47 Gerrit-Change-Number: 1048878 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de>

1 0

[Gerrit] ...core[master]: [IMPR] use BasePage.templates() with Page.templatesWithParams
by jenkins-bot (Code Review) 23 Jun '24

23 Jun '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048876?usp=email ) Change subject: [IMPR] use BasePage.templates() with Page.templatesWithParams ...................................................................... [IMPR] use BasePage.templates() with Page.templatesWithParams Page.templatesWithParams retrieves all templates. Therefore use BasePage.templates() instead of BasePage.itertemplates() to cache the templates. The caching was lost with 350760f in 9.2.0. Change-Id: I44006564eaaf260b72d6caaebb1c09d7a55c90c9 --- M pywikibot/page/_page.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/page/_page.py b/pywikibot/page/_page.py index 8dcda60..b89bc2c 100644 --- a/pywikibot/page/_page.py +++ b/pywikibot/page/_page.py @@ -78,7 +78,7 @@ # WARNING: may not return all templates used in particularly # intricate cases such as template substitution titles = {t.title() - for t in self.itertemplates(namespaces=Namespace.TEMPLATE)} + for t in self.templates(namespaces=Namespace.TEMPLATE)} templates = self.raw_extracted_templates # backwards-compatibility: convert the dict returned as the second # element into a list in the format used by old scripts -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048876?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I44006564eaaf260b72d6caaebb1c09d7a55c90c9 Gerrit-Change-Number: 1048876 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] ...core[master]: [tests] install wikitextparser for doctest
by Xqt (Code Review) 23 Jun '24

23 Jun '24

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048874?usp=email ) Change subject: [tests] install wikitextparser for doctest ...................................................................... [tests] install wikitextparser for doctest Bug: T368223 Change-Id: I59f77e50bc8959cbe70a675b497f41cab5afb711 --- M .github/workflows/doctest.yml 1 file changed, 1 insertion(+), 0 deletions(-) Approvals: Xqt: Verified; Looks good to me, approved diff --git a/.github/workflows/doctest.yml b/.github/workflows/doctest.yml index 6ec8aa6..56c5c54 100644 --- a/.github/workflows/doctest.yml +++ b/.github/workflows/doctest.yml @@ -63,6 +63,7 @@ pip install "PyMySQL >= 1.0.0" pip install pytest pip install "sseclient<0.0.23,>=0.0.18" + pip install wikitextparser - name: Generate user files run: | -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1048874?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I59f77e50bc8959cbe70a675b497f41cab5afb711 Gerrit-Change-Number: 1048874 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de>

1 0

[Gerrit] ...core[master]: [fix] re-implement Page.main_authors() using xtools
by jenkins-bot (Code Review) 23 Jun '24

23 Jun '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1037928?usp=email ) Change subject: [fix] re-implement Page.main_authors() using xtools ...................................................................... [fix] re-implement Page.main_authors() using xtools Additional parameters are used to filter the result. Bug: T366100 Change-Id: I9bed91ae63de6d2583b9e3ccb0970900e167b340 --- M pywikibot/page/_toolforge.py M requirements.txt M tests/wikiblame_tests.py M tox.ini 4 files changed, 193 insertions(+), 55 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/page/_toolforge.py b/pywikibot/page/_toolforge.py index ebdf5b2..b701c1f 100644 --- a/pywikibot/page/_toolforge.py +++ b/pywikibot/page/_toolforge.py @@ -11,9 +11,21 @@ import collections import re +from typing import TYPE_CHECKING import pywikibot -from pywikibot import config +from pywikibot import textlib +from pywikibot.tools import deprecated, deprecated_args + +try: + import wikitextparser +except ImportError as e: + wikitextparser = e + +if TYPE_CHECKING: + import datetime + from pywikibot import Timestamp + DATETYPE = str | Timestamp | datetime.datetime | datetime.date | None class WikiBlameMixin: @@ -24,7 +36,10 @@ """ #: Supported wikipedia site codes - WIKIBLAME_CODES = 'als', 'bar', 'de', 'en', 'it', 'nds', 'sco' + WIKIBLAME_CODES = ( + 'ar', 'de', 'en', 'es', 'eu', 'fr', 'hu', 'id', 'it', 'ja', 'nl', 'pl', + 'pt', 'tr', + ) def _check_wh_supported(self): """Check if WikiHistory is supported.""" @@ -44,68 +59,161 @@ if not self.exists(): raise pywikibot.exceptions.NoPageError(self) - def main_authors(self, *, - onlynew: bool | None = None) -> collections.Counter: - """Retrieve the 5 topmost main authors of an article. + if isinstance(wikitextparser, ImportError): + raise wikitextparser - This method uses WikiHistory to retrieve the text based main - authorship. + @deprecated('authorsship', since='9.3.0') + @deprecated_args(onlynew=None) # since 9.2.0 + def main_authors(self) -> collections.Counter[str, int]: + """Retrieve the 5 topmost main authors of an article. Sample: >>> import pywikibot - >>> site = pywikibot.Site('wikipedia:nds') - >>> page = pywikibot.Page(site, 'Python (Programmeerspraak)') - >>> auth = page.main_authors(onlynew=False) - >>> auth - Counter({'RebeccaBreu': 99, 'Slomox': 1}) + >>> site = pywikibot.Site('wikipedia:eu') + >>> page = pywikibot.Page(site, 'Python (informatika)') + >>> auth = page.main_authors() + >>> auth.most_common(1) + [('Ksarasola', 80)] - .. note:: Only implemented for main namespace pages. - .. note:: Only wikipedias of :attr:`WIKIBLAME_CODES` are supported. - .. attention:: This method does not return new results due to - :phab:`366100`. + .. important:: Only implemented for main namespace pages and + only wikipedias of :attr:`WIKIBLAME_CODES` are supported. .. seealso:: - https://wikihistory.toolforge.org - https://de.wikipedia.org/wiki/Wikipedia:Technik/Cloud/wikihistory + - https://xtools.wmcloud.org/authorship/ .. versionchanged:: 9.2 do not use any wait cycles due to :phab:`366100`. + .. versionchanged:: 9.3 + https://xtools.wmcloud.org/authorship/ is used to retrieve + authors + .. deprecated:: 9.3 + use :meth:`authorship` instead. - :param onlynew: Currently meaningless - :return: Number of edits for each username - :raise NotImplementedError: unsupported site or unsupported namespace - :raise pywikibot.exceptions.NoPageError: The page does not exist - :raise pywikibot.exceptions.TimeoutError: No cached results found + :return: Percentage of edits for each username + + :raise ImportError: missing ``wikitextparser`` module. + :raise NotImplementedError: unsupported site or unsupported + namespace. + :raise Error: Error response from xtools. + :raise NoPageError: The page does not exist. + :raise requests.exceptions.HTTPError: 429 Client Error: Too Many + Requests for url; login to meta family first. """ - baseurl = 'https://wikihistory.toolforge.org' - pattern = (r'><bdi>(?P<author>.+?)</bdi></a>\s' - r'\((?P<percent>\d{1,3})&') + return collections.Counter( + {user: int(cnt) for user, (_, cnt) in self.authorship(5).items()}) + + def authorship( + self, + n: int | None = None, + *, + min_chars: int = 0, + min_pct: float = 0.0, + max_pct_sum: float | None = None, + revid: int | None = None, + date: DATETYPE = None, + ) -> dict[str, tuple[int, float]]: + """Retrieve authorship attributon of an article. + + This method uses XTools/Authorship to retrieve the authors + measured by character count. + + Sample: + + >>> import pywikibot + >>> site = pywikibot.Site('wikipedia:en') + >>> page = pywikibot.Page(site, 'Pywikibot') + >>> auth = page.authorship() + >>> auth + {'1234qwer1234qwer4': (68, 100.0)} + + .. important:: Only implemented for main namespace pages and + only wikipedias of :attr:`WIKIBLAME_CODES` are supported. + .. seealso:: + - https://xtools.wmcloud.org/authorship/ + - https://www.mediawiki.org/wiki/XTools/Authorship + - https://www.mediawiki.org/wiki/WikiWho + + .. versionadded:: 9.3 + this method replaces :meth:`main_authors`. + + :param n: Only return the first *n* or fewer authors. + :param min_chars: Only return authors with more than *min_chars* + chars changes. + :param min_pct: Only return authors with more than *min_pct* + percentage edits. + :param max_pct_sum: Only return authors until the prcentage sum + reached *max_pct_sum*. + :param revid: The revision id for the authors should be found. + If ``None`` or ``0``, the latest revision is be used. Cannot + be used together with *date*. + :param date: The revision date for the authors should be found. + If ``None``, it will be ignored. Cannot be used together + with *revid*. If the parameter is a string it must be given + in the form ``YYYY-MM-DD`` + :return: Character count and percentage of edits for each + username. + + :raise ImportError: missing ``wikitextparser`` module + :raise NotImplementedError: unsupported site or unsupported + namespace. + :raise Error: Error response from xtools. + :raiseNoPageError: The page does not exist. + :raise requests.exceptions.HTTPError: 429 Client Error: Too Many + Requests for url; login to meta family first. + """ + baseurl = 'https://xtools.wmcloud.org/authorship/{url}&format=wikitext' + pattern = r'\[\[.+[|/](?P<user>.+)\]\]' self._check_wh_supported() - url = baseurl + '/wiki/getauthors.php?wiki={}wiki&page_id={}'.format( - self.site.code, self.pageid) - if onlynew: - url += '&onlynew=1' + if revid and date: + raise ValueError( + 'You cannot specify revid together with date argument') - for current_retries in range(config.max_retries): - r = pywikibot.comms.http.fetch(url) - if r.status_code != 200: - r.raise_for_status() + if date is None: + show = revid or 0 + else: + show = str(date)[:10] - if 'Timeout' not in r.text: # window.setTimeout in result - return collections.Counter( - {user: int(cnt) - for user, cnt in re.findall(pattern, r.text)}) + url = '{}.wikipedia.org/{}/{}?uselang={}'.format( + self.site.code, + self.title(as_url=True, with_ns=False, with_section=False), + show, + 'en', + ) + url = baseurl.format(url=url) - break # T366100 + r = pywikibot.comms.http.fetch(url) + if r.status_code != 200: + r.raise_for_status() - delay = pywikibot.config.retry_wait * 2 ** current_retries - pywikibot.warning('WikiHistory timeout.\n' - f'Waiting {delay:.1f} seconds before retrying.') - pywikibot.sleep(delay) - if onlynew is None and current_retries >= config.max_retries - 2: - url += '&onlynew=1' + result: list[list[str]] = [] + try: + table = wikitextparser.parse(r.text).tables[0] + except IndexError: + pattern = textlib.get_regexes('code')[0] + msg = pattern.search(r.text)[0] + raise pywikibot.exceptions.Error(textlib.removeHTMLParts(msg)) - raise pywikibot.exceptions.TimeoutError( - 'Maximum retries attempted without success.') + pct_sum = 0.0 + for row in table.data(): + if row[0] == 'Rank': + continue # skip headline + + rank = int(row[0]) + user = re.match(pattern, row[1])['user'] + chars = int(row[3].replace(',', '_')) + percent = float(row[4].rstrip('%')) + + # take into account tht data() is ordered + if n and rank > n or chars < min_chars or percent < min_pct: + break + + result.append((user, chars, percent)) + pct_sum += percent + if max_pct_sum and pct_sum >= max_pct_sum: + break + + return {user: (chars, percent) for user, chars, percent in result} diff --git a/requirements.txt b/requirements.txt index 5da319c..9120e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,8 @@ # MediaWiki markup parser # mwparserfromhell is mandatory but wikitextparser can be used instead # mwparserfromhell is still required for commons_information.py and patrol.py -# wikitextparser>=0.47.5 +# wikitextparser is required for Page.authorship() +wikitextparser>=0.47.5 # OAuth support # mwoauth 0.2.4 is needed because it supports getting identity information diff --git a/tests/wikiblame_tests.py b/tests/wikiblame_tests.py index 3789b6a..0d5057c 100644 --- a/tests/wikiblame_tests.py +++ b/tests/wikiblame_tests.py @@ -1,16 +1,17 @@ """Tests for the WikiHistoryMixin.""" # -# (C) Pywikibot team, 2022-2023 +# (C) Pywikibot team, 2022-2024 # # Distributed under the terms of the MIT license. # from __future__ import annotations +import re import unittest from contextlib import suppress import pywikibot -from tests.aspects import TestCase +from tests.aspects import TestCase, require_modules class TestWikiBlameMixin(TestCase): @@ -18,17 +19,44 @@ """Test WikiBlameMixin using nds wiki.""" family = 'wikipedia' - code = 'nds' + code = 'nl' + def test_exceptions(self): + """Test that main_authors fails if page does not exist.""" + page = pywikibot.Page(self.site, 'Pywikibot') + title = re.escape(page.title(as_link=True)) + with self.assertRaisesRegex(pywikibot.exceptions.NoPageError, + f"Page {title} doesn't exist"): + page.authorship() + + page = pywikibot.Page(self.site, 'Project:Pywikibot') + with self.assertRaisesRegex( + NotImplementedError, + 'main_authors method is implemented for main namespace only'): + page.authorship() + + @require_modules('wikitextparser') def test_main_authors(self): """Test main_authors() method.""" - page = pywikibot.Page(self.site, 'Python (Programmeerspraak)') - auth = page.main_authors(onlynew=False) + page = pywikibot.Page(self.site, 'Python (programmeertaal)') + auth = page.authorship(5) self.assertLessEqual(len(auth), 5) - self.assertLessEqual(sum(auth.values()), 100) - user, value = auth.most_common(1)[0] - self.assertEqual(user, 'RebeccaBreu') - self.assertGreater(value, 0) + self.assertLessEqual(sum(pct for _, pct in auth.values()), 100) + user, values = next(iter(auth.items())) + self.assertEqual(user, 'Emperor045') + self.assertIsInstance(values[0], int) + self.assertIsInstance(values[1], float) + + @require_modules('wikitextparser') + def test_restrictions(self): + """Test main_authors() method with restrictions.""" + page = pywikibot.Page(pywikibot.Site('wikipedia:en'), 'Python') + auth = page.authorship(min_chars=100, min_pct=5.0) + self.assertLessEqual(len(auth), 4) + for k, (chars, pct) in auth.items(): + with self.subTest(user=k): + self.assertGreaterEqual(chars, 100) + self.assertGreaterEqual(pct, 5.0) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini index c254410..f11f8dd 100644 --- a/tox.ini +++ b/tox.ini @@ -82,6 +82,7 @@ deps = pytest >= 7.0.1 + wikitextparser .[eventstreams] .[mysql] -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1037928?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I9bed91ae63de6d2583b9e3ccb0970900e167b340 Gerrit-Change-Number: 1037928 Gerrit-PatchSet: 14 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits