jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1123799?usp=email )

Change subject: [bugfix] use googlesearch-python for GoogleSearchPageGenerator ......................................................................

[bugfix] use googlesearch-python for GoogleSearchPageGenerator

google package is unsupported for more than 4 years. The newest version fails because the module name was changed. The GoogleSearchPageGenerator fails for years due to changed protocol from http to https.

- use googlesearch-python package - add set_maximum_items method to GoogleSearchPageGenerator to be used with pagegenerators.GeneratorFactory - total argument was added; the default value is 10 - queryGoogle method got **kwargs to pass additional arguments to the googlesearch-python package

Bug: T387618 Change-Id: Ic3b594dc7b1f8510691274202050869abe915100 --- M pywikibot/pagegenerators/_generators.py M requirements.txt M setup.py 3 files changed, 93 insertions(+), 48 deletions(-)

Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/pagegenerators/_generators.py b/pywikibot/pagegenerators/_generators.py index 4200291..2cf3c0e 100644 --- a/pywikibot/pagegenerators/_generators.py +++ b/pywikibot/pagegenerators/_generators.py @@ -10,7 +10,6 @@ import codecs import io import re -import sys import typing from collections import abc from functools import partial @@ -872,85 +871,131 @@ yield page

-# following classes just ported from version 1 without revision; not tested - - class GoogleSearchPageGenerator(GeneratorWrapper):

"""Page generator using Google search results.

- To use this generator, you need to install the package 'google': + To use this generator, you need to install the googlesearch package::

- :py:obj:`https://pypi.org/project/google%60 - - This package has been available since 2010, hosted on GitHub - since 2012, and provided by PyPI since 2013. + pip install googlesearch-python

As there are concerns about Google's Terms of Service, this generator prints a warning for each query.

+ .. seealso:: https://policies.google.com/terms .. versionchanged:: 7.6 subclassed from :class:`tools.collections.GeneratorWrapper` + .. versionchanged:: 10.1 + ``googlesearch-python`` package is needed instead of ``google``, + see :phab:`T387618` for further informations. The *total* + parameter was added. The *query* parameter is positional only. + All other parameters are keyword only. """

- def __init__(self, query: str | None = None, - site: BaseSite | None = None) -> None: + def __init__(self, query: str = '', /, *, + site: BaseSite | None = None, + total: int = 10) -> None: """Initializer.

+ :param query: the text to search for. :param site: Site for generator results. + :param total: the maximum number of changes to return, default + is 10 which is also set by googlesearch package. """ - self.query = query or pywikibot.input('Please enter the search query:') - if site is None: - site = pywikibot.Site() - self.site = site - self._google_query = None + self.query = query or pywikibot.input( + 'Please enter the search query:') + self.site = site or pywikibot.Site() + self.limit = total

@staticmethod - def queryGoogle(query: str) -> Generator[str, None, None]: - """Perform a query using python package 'google'. + def queryGoogle(query: str, /, **kwargs) -> Generator[str, None, None]: + """Perform a query using ``googlesearch-python`` package.

- The terms of service as at June 2014 give two conditions that - may apply to use of search: + .. admonition:: Terms of Service

- 1. Don't access [Google Services] using a method other than - the interface and the instructions that [they] provide. - 2. Don't remove, obscure, or alter any legal notices - displayed in or along with [Google] Services. + The terms of service as at June 2014 give two conditions that + may apply to use of search:

- Both of those issues should be managed by the package 'google', - however Pywikibot will at least ensure the user sees the TOS - in order to comply with the second condition. + 1. Don't access [Google Services] using a method other than + the interface and the instructions that [they] provide. + 2. Don't remove, obscure, or alter any legal notices + displayed in or along with [Google] Services. + + Both of those issues should be managed by the + ``googlesearch-python`` package, however Pywikibot will at + least ensure the user sees the TOS in order to comply with + the second. + + .. seealso:: https://policies.google.com/terms + condition. + + .. important:: These note are from 2014 and have not been + reviewed or updated since then. + + .. versionchanged:: 10.1 + *query* is positional only; *kwargs* parameter was added. + + :param query: the text to search for. + :param kwargs: other keyword arguments passed to ``googlesearch`` + module. """ try: - import google - except ImportError: - pywikibot.error('generator GoogleSearchPageGenerator ' - "depends on package 'google'.\n" - 'To install, please run: pip install google.') - sys.exit(1) + import googlesearch + except ModuleNotFoundError: + pywikibot.error("""\ +generator GoogleSearchPageGenerator depends on package +'googlesearch-python'. To install, please run: + + pip install googlesearch-python""") + return + pywikibot.warning('Please read http://www.google.com/accounts/TOS') - yield from google.search(query) + yield from googlesearch.search(query, **kwargs)

@property def generator(self) -> Generator[pywikibot.page.Page, None, None]: """Yield results from :meth:`queryGoogle` query.

- Google contains links in the format: - https://de.wikipedia.org/wiki/en:Foobar - .. versionchanged:: 7.6 changed from iterator method to generator property + .. versionchanged:: 10.1 + use :meth:`site.protocol + <pywikibot.site._basesite.BaseSite.protocol>` to get the base + URL. Also filter duplicates. """ + if not self.query: + pywikibot.warning('No query string was specified') + return + # restrict query to local site - local_query = f'{self.query} site:{self.site.hostname()}' - base = f'http://%7Bself.site.hostname()%7D%7Bself.site.articlepath%7D' - pattern = base.replace('{}', '(.+)') - for url in self.queryGoogle(local_query): - m = re.search(pattern, url) - if m: - page = pywikibot.Page(pywikibot.Link(m[1], self.site)) - if page.site == self.site: - yield page + site = self.site + local_query = f'{self.query} site:{site.hostname()}' + base = f'{site.protocol()}://{site.hostname()}{site.articlepath}' + pattern = re.compile(base.replace('{}', '(?P<title>.+)')) + + for url in self.queryGoogle(local_query, num_results=self.limit, + unique=True): + m = pattern.fullmatch(url) + if not m: + continue + + page = pywikibot.Page(pywikibot.Link(m['title'], site)) + + # Google may contain links in the format: + # https://de.wikipedia.org/wiki/en:Foobar + if page.site == site: + yield page + + def set_maximum_items(self, value: int, /): + """Set the maximum number of items to be retrieved from google. + + This method is added to be used by the + :class:`pagegenerators.GeneratorFactory` to circumvent call of + :func:`itertools.islice` filter for this generator. + + .. versionadded:: 10.1 + """ + self.limit = value

def MySQLPageGenerator(query: str, site: BaseSite | None = None, diff --git a/requirements.txt b/requirements.txt index b096bdd..c6f3ef3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ Pillow==10.4.0; python_version < "3.9"

# core pagegenerators -google >= 1.7 +googlesearch-python >= 1.3.0 requests-sse >= 0.5.0

# The mysql generator in pagegenerators depends on PyMySQL diff --git a/setup.py b/setup.py index 4e40ecb..f25a3d1 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ 'eventstreams': ['requests-sse>=0.5.0'], 'isbn': ['python-stdnum>=1.20'], 'Graphviz': ['pydot>=3.0.2'], - 'Google': ['google>=1.7'], + 'Google': ['googlesearch-python >= 1.3.0'], 'memento': ['memento_client==0.6.1'], 'wikitextparser': ['wikitextparser>=0.56.3'], 'mysql': ['PyMySQL >= 1.1.1'],