[Pywikibot-commits] [Gerrit] ...core[master]: [IMPR] move threading parts to tools.threading.py (part 3)

Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/834652 ) Change subject: [IMPR] move threading parts to tools.threading.py (part 3) ...................................................................... [IMPR] move threading parts to tools.threading.py (part 3) - move threading classes from tools to tools.threading - update usage in ui.terminal_interface_base.py and weblinkchecker.py - rename thread test to tools_threading - update documentation accordingly Change-Id: I493f5362670f57326ceef38e43ca172179d1eef0 --- M ROADMAP.rst M docs/api_ref/pywikibot.tools.rst M docs/scripts/index.rst M docs/tests_ref/index.rst D docs/tests_ref/thread_tests.rst A docs/tests_ref/tools_threading_tests.rst M pywikibot/CONTENT.rst M pywikibot/tools/__init__.py M pywikibot/tools/threading.py M pywikibot/userinterfaces/terminal_interface_base.py M scripts/weblinkchecker.py M tests/__init__.py R tests/tools_threading_tests.py 13 files changed, 42 insertions(+), 1,015 deletions(-) Approvals: jenkins-bot: Verified Xqt: Verified; Looks good to me, approved diff --git a/ROADMAP.rst b/ROADMAP.rst index 947c5a8..d94a213 100644 --- a/ROADMAP.rst +++ b/ROADMAP.rst @@ -1,6 +1,7 @@ Current release 7.7.0 ^^^^^^^^^^^^^^^^^^^^^ +* tools' threading classes were moved to :mod:`tools.threading` submodule * OmegaWiki family was removed * Provide global ``-config`` option to specify the user config file name * Run :mod:`pywikibot.scripts.login` script in parallel tasks if ``-async`` option is given (:phab:`T57899`) @@ -14,6 +15,7 @@ Deprecations ^^^^^^^^^^^^ +* 7.7.0: :mod:`tools.threading` classes should no longer imported from :mod:`tools` * 7.6.0: :mod:`tools.itertools` datatypes should no longer imported from :mod:`tools` * 7.6.0: :mod:`tools.collections` datatypes should no longer imported from :mod:`tools` * 7.5.0: :mod:`textlib`.tzoneFixedOffset class will be removed in favour of :class:`time.TZoneFixedOffset` diff --git a/docs/api_ref/pywikibot.tools.rst b/docs/api_ref/pywikibot.tools.rst index 8ed647e..28289da 100644 --- a/docs/api_ref/pywikibot.tools.rst +++ b/docs/api_ref/pywikibot.tools.rst @@ -40,6 +40,12 @@ .. automodule:: tools.itertools :synopsis: Iterator functions +:mod:`tools.threading` --- Thread-based Classes +----------------------------------------------- + +.. automodule:: tools.threading + :synopsis: Threading classes + :mod:`tools.\_logging` --- logging.Formatter Subclass ----------------------------------------------------- diff --git a/docs/scripts/index.rst b/docs/scripts/index.rst index 1f374ed..c20b9a5 100644 --- a/docs/scripts/index.rst +++ b/docs/scripts/index.rst @@ -1,5 +1,5 @@ -Scripts package ---------------- +Scripts Package Description +--------------------------- .. automodule:: scripts :no-members: @@ -8,7 +8,7 @@ :start-after: included. :end-before: More precise -Scripts descriptions +Scripts Descriptions -------------------- .. toctree:: diff --git a/docs/tests_ref/index.rst b/docs/tests_ref/index.rst index a3218d2..9f92532 100644 --- a/docs/tests_ref/index.rst +++ b/docs/tests_ref/index.rst @@ -70,12 +70,12 @@ tests<./tests_tests> textlib<./textlib_tests> thanks<./thanks_tests> - thread<./thread_tests> time<./time_tests> timestripper<./timestripper_tests> tools_chars<./tools_chars_tests> tools_deprecate<./tools_deprecate_tests> tools_formatter<./tools_formatter_tests> + threading<./tools_threading_tests> tools<./tools_tests> ui_options<./ui_options_tests> ui<./ui_tests> diff --git a/docs/tests_ref/thread_tests.rst b/docs/tests_ref/thread_tests.rst deleted file mode 100644 index 578acc7..0000000 --- a/docs/tests_ref/thread_tests.rst +++ /dev/null @@ -1,7 +0,0 @@ -tests.thread\_tests module -========================== - -.. automodule:: tests.thread_tests - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/tests_ref/tools_threading_tests.rst b/docs/tests_ref/tools_threading_tests.rst new file mode 100644 index 0000000..d68846a --- /dev/null +++ b/docs/tests_ref/tools_threading_tests.rst @@ -0,0 +1,7 @@ +tests.tools\_threading\_tests module +==================================== + +.. automodule:: tests.tools_threading_tests + :members: + :undoc-members: + :show-inheritance: diff --git a/pywikibot/CONTENT.rst b/pywikibot/CONTENT.rst index 096ce26..e92d50b 100644 --- a/pywikibot/CONTENT.rst +++ b/pywikibot/CONTENT.rst @@ -235,6 +235,8 @@ +----------------------------+------------------------------------------------------+ | itertools.py | Iterator functions | +----------------------------+------------------------------------------------------+ + | threading.py | Threading classes | + +----------------------------+------------------------------------------------------+ +-----------------------------------------------------------------------------------+ diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py index 595ea01..5bdf198 100644 --- a/pywikibot/tools/__init__.py +++ b/pywikibot/tools/__init__.py @@ -8,13 +8,10 @@ import hashlib import ipaddress import os -import queue import re import stat import subprocess import sys -import threading -import time from contextlib import suppress from functools import total_ordering, wraps @@ -62,6 +59,7 @@ __all__ = ( + # deprecating functions 'ModuleDeprecationWrapper', 'add_decorated_full_name', 'add_full_name', @@ -74,6 +72,7 @@ 'redirect_func', 'remove_last_args', + # other tools 'PYTHON_VERSION', 'is_ip_address', 'has_module', @@ -86,9 +85,6 @@ 'normalize_username', 'Version', 'MediaWikiVersion', - 'RLock', - 'ThreadedGenerator', - 'ThreadList', 'SelfCallMixin', 'SelfCallDict', 'SelfCallString', @@ -475,214 +471,6 @@ return self._dev_version < other._dev_version -class RLock: - """Context manager which implements extended reentrant lock objects. - - This RLock is implicit derived from threading.RLock but provides a - locked() method like in threading.Lock and a count attribute which - gives the active recursion level of locks. - - Usage: - - >>> from pywikibot.tools import RLock - >>> lock = RLock() - >>> lock.acquire() - True - >>> with lock: print(lock.count) # nested lock - 2 - >>> lock.locked() - True - >>> lock.release() - >>> lock.locked() - False - - .. versionadded:: 6.2 - """ - - def __init__(self, *args, **kwargs) -> None: - """Initializer.""" - self._lock = threading.RLock(*args, **kwargs) - self._block = threading.Lock() - - def __enter__(self): - """Acquire lock and call atenter.""" - return self._lock.__enter__() - - def __exit__(self, *exc): - """Call atexit and release lock.""" - return self._lock.__exit__(*exc) - - def __getattr__(self, name): - """Delegate attributes and methods to self._lock.""" - return getattr(self._lock, name) - - def __repr__(self) -> str: - """Representation of tools.RLock instance.""" - return repr(self._lock).replace( - '_thread.RLock', - '{cls.__module__}.{cls.__class__.__name__}'.format(cls=self)) - - @property - def count(self): - """Return number of acquired locks.""" - with self._block: - counter = re.search(r'count=(\d+) ', repr(self)) - return int(counter.group(1)) - - def locked(self): - """Return true if the lock is acquired.""" - with self._block: - status = repr(self).split(maxsplit=1)[0][1:] - assert status in ('locked', 'unlocked') - return status == 'locked' - - -class ThreadedGenerator(threading.Thread): - - """Look-ahead generator class. - - Runs a generator in a separate thread and queues the results; can - be called like a regular generator. - - Subclasses should override self.generator, *not* self.run - - Important: the generator thread will stop itself if the generator's - internal queue is exhausted; but, if the calling program does not use - all the generated values, it must call the generator's stop() method to - stop the background thread. Example usage: - - >>> gen = ThreadedGenerator(target=range, args=(20,)) - >>> try: - ... data = list(gen) - ... finally: - ... gen.stop() - >>> data - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] - - .. versionadded:: 3.0 - """ - - def __init__(self, group=None, target=None, name: str = 'GeneratorThread', - args=(), kwargs=None, qsize: int = 65536) -> None: - """Initializer. Takes same keyword arguments as threading.Thread. - - target must be a generator function (or other callable that returns - an iterable object). - - :param qsize: The size of the lookahead queue. The larger the qsize, - the more values will be computed in advance of use (which can eat - up memory and processor time). - """ - if kwargs is None: - kwargs = {} - if target: - self.generator = target - if not hasattr(self, 'generator'): - raise RuntimeError('No generator for ThreadedGenerator to run.') - self.args, self.kwargs = args, kwargs - super().__init__(group=group, name=name) - self.queue = queue.Queue(qsize) - self.finished = threading.Event() - - def __iter__(self): - """Iterate results from the queue.""" - if not self.is_alive() and not self.finished.is_set(): - self.start() - # if there is an item in the queue, yield it, otherwise wait - while not self.finished.is_set(): - try: - yield self.queue.get(True, 0.25) - except queue.Empty: - pass - except KeyboardInterrupt: - self.stop() - - def stop(self) -> None: - """Stop the background thread.""" - self.finished.set() - - def run(self) -> None: - """Run the generator and store the results on the queue.""" - iterable = any(hasattr(self.generator, key) - for key in ('__iter__', '__getitem__')) - if iterable and not self.args and not self.kwargs: - self.__gen = self.generator - else: - self.__gen = self.generator(*self.args, **self.kwargs) - for result in self.__gen: - while True: - if self.finished.is_set(): - return - try: - self.queue.put_nowait(result) - except queue.Full: - time.sleep(0.25) - continue - break - # wait for queue to be emptied, then kill the thread - while not self.finished.is_set() and not self.queue.empty(): - time.sleep(0.25) - self.stop() - - -class ThreadList(list): - - """A simple threadpool class to limit the number of simultaneous threads. - - Any threading.Thread object can be added to the pool using the append() - method. If the maximum number of simultaneous threads has not been reached, - the Thread object will be started immediately; if not, the append() call - will block until the thread is able to start. - - >>> pool = ThreadList(limit=10) - >>> def work(): - ... time.sleep(1) - ... - >>> for x in range(20): - ... pool.append(threading.Thread(target=work)) - ... - - """ - - def __init__(self, limit: int = 128, wait_time: float = 2, *args) -> None: - """Initializer. - - :param limit: the number of simultaneous threads - :param wait_time: how long to wait if active threads exceeds limit - """ - self.limit = limit - self.wait_time = wait_time - super().__init__(*args) - for item in self: - if not isinstance(item, threading.Thread): - raise TypeError("Cannot add '{}' to ThreadList" - .format(type(item))) - - def active_count(self): - """Return the number of alive threads and delete all non-alive ones.""" - cnt = 0 - for item in self[:]: - if item.is_alive(): - cnt += 1 - else: - self.remove(item) - return cnt - - def append(self, thd): - """Add a thread to the pool and start it.""" - if not isinstance(thd, threading.Thread): - raise TypeError("Cannot append '{}' to ThreadList" - .format(type(thd))) - - while self.active_count() >= self.limit: - time.sleep(self.wait_time) - - super().append(thd) - thd.start() - pywikibot.logging.debug("thread {} ('{}') started" - .format(len(self), type(thd))) - - class SelfCallMixin: """ @@ -1010,3 +798,17 @@ 'filter_unique', replacement_name='pywikibot.tools.itertools.filter_unique', since='7.6.0') + +# Deprecate objects which has to be imported from tools.threading instead +wrapper.add_deprecated_attr( + 'RLock', + replacement_name='pywikibot.tools.threading.RLock', + since='7.7.0') +wrapper.add_deprecated_attr( + 'ThreadedGenerator', + replacement_name='pywikibot.tools.threading.ThreadedGenerator', + since='7.7.0') +wrapper.add_deprecated_attr( + 'ThreadList', + replacement_name='pywikibot.tools.threading.ThreadList', + since='7.7.0') diff --git a/pywikibot/tools/threading.py b/pywikibot/tools/threading.py index 595ea01..c4d997c 100644 --- a/pywikibot/tools/threading.py +++ b/pywikibot/tools/threading.py @@ -4,477 +4,21 @@ # # Distributed under the terms of the MIT license. # -import gzip -import hashlib -import ipaddress -import os import queue import re -import stat -import subprocess -import sys import threading import time -from contextlib import suppress -from functools import total_ordering, wraps -from importlib import import_module -from types import TracebackType -from typing import Any, Optional, Type -from warnings import catch_warnings, showwarning, warn - -import pkg_resources - import pywikibot # T306760 -from pywikibot.backports import Callable -from pywikibot.tools._deprecate import ( - ModuleDeprecationWrapper, - add_decorated_full_name, - add_full_name, - deprecate_arg, - deprecated, - deprecated_args, - get_wrapper_depth, - issue_deprecation_warning, - manage_wrapping, - redirect_func, - remove_last_args, -) -from pywikibot.tools._unidata import _first_upper_exception - - -pkg_Version = pkg_resources.packaging.version.Version # noqa: N816 - -try: - import bz2 -except ImportError as bz2_import_error: - try: - import bz2file as bz2 - warn('package bz2 was not found; using bz2file', ImportWarning) - except ImportError: - warn('package bz2 and bz2file were not found', ImportWarning) - bz2 = bz2_import_error - -try: - import lzma -except ImportError as lzma_import_error: - lzma = lzma_import_error __all__ = ( - 'ModuleDeprecationWrapper', - 'add_decorated_full_name', - 'add_full_name', - 'deprecate_arg', - 'deprecated', - 'deprecated_args', - 'get_wrapper_depth', - 'issue_deprecation_warning', - 'manage_wrapping', - 'redirect_func', - 'remove_last_args', - - 'PYTHON_VERSION', - 'is_ip_address', - 'has_module', - 'classproperty', - 'suppress_warnings', - 'ComparableMixin', - 'first_lower', - 'first_upper', - 'strtobool', - 'normalize_username', - 'Version', - 'MediaWikiVersion', 'RLock', 'ThreadedGenerator', 'ThreadList', - 'SelfCallMixin', - 'SelfCallDict', - 'SelfCallString', - 'open_archive', - 'merge_unique_dicts', - 'file_mode_checker', - 'compute_file_hash', - 'cached', ) -PYTHON_VERSION = sys.version_info[:3] - - -def is_ip_address(value: str) -> bool: - """Check if a value is a valid IPv4 or IPv6 address. - - .. versionadded:: 6.1 - Was renamed from ``is_IP()``. - - :param value: value to check - """ - with suppress(ValueError): - ipaddress.ip_address(value) - return True - - return False - - -def has_module(module, version=None) -> bool: - """Check if a module can be imported. - - .. versionadded:: 3.0 - - .. versionchanged:: 6.1 - Dependency of distutils was dropped because the package will be - removed with Python 3.12. - """ - try: - m = import_module(module) - except ImportError: - return False - if version: - if not hasattr(m, '__version__'): - return False - - required_version = pkg_resources.parse_version(version) - module_version = pkg_resources.parse_version(m.__version__) - - if module_version < required_version: - warn('Module version {} is lower than requested version {}' - .format(module_version, required_version), ImportWarning) - return False - - return True - - -class classproperty: # noqa: N801 - - """ - Descriptor class to access a class method as a property. - - This class may be used as a decorator:: - - class Foo: - - _bar = 'baz' # a class property - - @classproperty - def bar(cls): # a class property method - return cls._bar - - Foo.bar gives 'baz'. - - .. versionadded:: 3.0 - """ - - def __init__(self, cls_method) -> None: - """Hold the class method.""" - self.method = cls_method - self.__doc__ = self.method.__doc__ - - def __get__(self, instance, owner): - """Get the attribute of the owner class by its method.""" - return self.method(owner) - - -class suppress_warnings(catch_warnings): # noqa: N801 - - """A decorator/context manager that temporarily suppresses warnings. - - Those suppressed warnings that do not match the parameters will be raised - shown upon exit. - - .. versionadded:: 3.0 - """ - - def __init__( - self, - message: str = '', - category=Warning, - filename: str = '' - ) -> None: - """Initialize the object. - - The parameter semantics are similar to those of - `warnings.filterwarnings`. - - :param message: A string containing a regular expression that the start - of the warning message must match. (case-insensitive) - :param category: A class (a subclass of Warning) of which the warning - category must be a subclass in order to match. - :type category: type - :param filename: A string containing a regular expression that the - start of the path to the warning module must match. - (case-sensitive) - """ - self.message_match = re.compile(message, re.I).match - self.category = category - self.filename_match = re.compile(filename).match - super().__init__(record=True) - - def __enter__(self) -> None: - """Catch all warnings and store them in `self.log`.""" - self.log = super().__enter__() - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType] - ) -> None: - """Stop logging warnings and show those that do not match to params.""" - super().__exit__(exc_type, exc_val, exc_tb) - for warning in self.log: - if ( - not issubclass(warning.category, self.category) - or not self.message_match(str(warning.message)) - or not self.filename_match(warning.filename) - ): - showwarning( - warning.message, warning.category, warning.filename, - warning.lineno, warning.file, warning.line) - - def __call__(self, func): - """Decorate func to suppress warnings.""" - @wraps(func) - def suppressed_func(*args, **kwargs): - with self: - return func(*args, **kwargs) - return suppressed_func - - -# From http://python3porting.com/preparing.html -class ComparableMixin: - - """Mixin class to allow comparing to other objects which are comparable. - - .. versionadded:: 3.0 - """ - - def __lt__(self, other): - """Compare if self is less than other.""" - return other > self._cmpkey() - - def __le__(self, other): - """Compare if self is less equals other.""" - return other >= self._cmpkey() - - def __eq__(self, other): - """Compare if self is equal to other.""" - return other == self._cmpkey() - - def __ge__(self, other): - """Compare if self is greater equals other.""" - return other <= self._cmpkey() - - def __gt__(self, other): - """Compare if self is greater than other.""" - return other < self._cmpkey() - - def __ne__(self, other): - """Compare if self is not equal to other.""" - return other != self._cmpkey() - - -def first_lower(string: str) -> str: - """ - Return a string with the first character uncapitalized. - - Empty strings are supported. The original string is not changed. - - .. versionadded:: 3.0 - """ - return string[:1].lower() + string[1:] - - -def first_upper(string: str) -> str: - """ - Return a string with the first character capitalized. - - Empty strings are supported. The original string is not changed. - - .. versionadded:: 3.0 - - .. note:: MediaWiki doesn't capitalize - some characters the same way as Python. - This function tries to be close to - MediaWiki's capitalize function in - title.php. See T179115 and T200357. - """ - first = string[:1] - return (_first_upper_exception(first) or first.upper()) + string[1:] - - -def strtobool(val: str) -> bool: - """Convert a string representation of truth to True or False. - - This is a reimplementation of distutils.util.strtobool due to - :pep:`632#Migration Advice` - - .. versionadded:: 7.1 - - :param val: True values are 'y', 'yes', 't', 'true', 'on', and '1'; - false values are 'n', 'no', 'f', 'false', 'off', and '0'. - :raises ValueError: `val` is not a valid truth value - """ - val = val.lower() - if val in ('y', 'yes', 't', 'true', 'on', '1'): - return True - if val in ('n', 'no', 'f', 'false', 'off', '0'): - return False - raise ValueError('invalid truth value {!r}'.format(val)) - - -def normalize_username(username) -> Optional[str]: - """Normalize the username. - - .. versionadded:: 3.0 - """ - if not username: - return None - username = re.sub('[_ ]+', ' ', username).strip() - return first_upper(username) - - -class Version(pkg_Version): - - """Version from pkg_resouce vendor package. - - This Version provides propreties of vendor package 20.4 shipped with - setuptools 49.4.0. - - .. versionadded:: 6.4 - """ - - def __getattr__(self, name): - """Provides propreties of vendor package 20.4.""" - if name in ('epoch', 'release', 'pre', ): - return getattr(self._version, name) - if name in ('post', 'dev'): - attr = getattr(self._version, name) - return attr[1] if attr else None - if name == 'is_devrelease': - return self.dev is not None - - parts = ('major', 'minor', 'micro') - try: - index = parts.index(name) - except ValueError: - raise AttributeError('{!r} object has to attribute {!r}' - .format(type(self).__name__, name)) from None - release = self.release - return release[index] if len(release) >= index + 1 else 0 - - -@total_ordering -class MediaWikiVersion: - - """ - Version object to allow comparing 'wmf' versions with normal ones. - - The version mainly consist of digits separated by periods. After that is a - suffix which may only be 'wmf<number>', 'alpha', 'beta<number>' or - '-rc.<number>' (the - and . are optional). They are considered from old to - new in that order with a version number without suffix is considered the - newest. This secondary difference is stored in an internal _dev_version - attribute. - - Two versions are equal if their normal version and dev version are equal. A - version is greater if the normal version or dev version is greater. For - example:: - - 1.34 < 1.34.1 < 1.35wmf1 < 1.35alpha < 1.35beta1 < 1.35beta2 - < 1.35-rc-1 < 1.35-rc.2 < 1.35 - - Any other suffixes are considered invalid. - - .. versionadded:: 3.0 - - .. versionchanged:: 6.1 - Dependency of distutils was dropped because the package will be - removed with Python 3.12. - """ - - MEDIAWIKI_VERSION = re.compile( - r'(\d+(?:\.\d+)+)(-?wmf\.?(\d+)|alpha|beta(\d+)|-?rc\.?(\d+)|.*)?$') - - def __init__(self, version_str: str) -> None: - """ - Initializer. - - :param version_str: version to parse - """ - self._parse(version_str) - - def _parse(self, version_str: str) -> None: - version_match = MediaWikiVersion.MEDIAWIKI_VERSION.match(version_str) - - if not version_match: - raise ValueError('Invalid version number "{}"'.format(version_str)) - - components = [int(n) for n in version_match.group(1).split('.')] - - # The _dev_version numbering scheme might change. E.g. if a stage - # between 'alpha' and 'beta' is added, 'beta', 'rc' and stable releases - # are reassigned (beta=3, rc=4, stable=5). - - if version_match.group(3): # wmf version - self._dev_version = (0, int(version_match.group(3))) - elif version_match.group(4): - self._dev_version = (2, int(version_match.group(4))) - elif version_match.group(5): - self._dev_version = (3, int(version_match.group(5))) - elif version_match.group(2) in ('alpha', '-alpha'): - self._dev_version = (1, ) - else: - for handled in ('wmf', 'alpha', 'beta', 'rc'): - # if any of those pops up here our parser has failed - assert handled not in version_match.group(2), \ - 'Found "{}" in "{}"'.format(handled, - version_match.group(2)) - if version_match.group(2): - pywikibot.logging.debug('Additional unused version part ' - '"{}"'.format(version_match.group(2))) - self._dev_version = (4, ) - - self.suffix = version_match.group(2) or '' - self.version = tuple(components) - - @staticmethod - def from_generator(generator: str) -> 'MediaWikiVersion': - """Create instance from a site's generator attribute.""" - prefix = 'MediaWiki ' - - if not generator.startswith(prefix): - raise ValueError('Generator string ({!r}) must start with ' - '"{}"'.format(generator, prefix)) - - return MediaWikiVersion(generator[len(prefix):]) - - def __str__(self) -> str: - """Return version number with optional suffix.""" - return '.'.join(str(v) for v in self.version) + self.suffix - - def __eq__(self, other: Any) -> bool: - if isinstance(other, str): - other = MediaWikiVersion(other) - elif not isinstance(other, MediaWikiVersion): - return False - - return self.version == other.version and \ - self._dev_version == other._dev_version - - def __lt__(self, other: Any) -> bool: - if isinstance(other, str): - other = MediaWikiVersion(other) - elif not isinstance(other, MediaWikiVersion): - raise TypeError("Comparison between 'MediaWikiVersion' and '{}' " - 'unsupported'.format(type(other).__name__)) - - if self.version != other.version: - return self.version < other.version - return self._dev_version < other._dev_version - - class RLock: """Context manager which implements extended reentrant lock objects. @@ -484,7 +28,6 @@ Usage: - >>> from pywikibot.tools import RLock True @@ -681,332 +224,3 @@ thd.start() pywikibot.logging.debug("thread {} ('{}') started" .format(len(self), type(thd))) - - -class SelfCallMixin: - - """ - Return self when called. - - When '_own_desc' is defined it'll also issue a deprecation warning using - issue_deprecation_warning('Calling ' + _own_desc, 'it directly'). - - .. versionadded:: 3.0 - .. deprecated:: 6.2 - """ - - def __call__(self): - """Do nothing and just return itself.""" - issue_deprecation_warning('Referencing this attribute like a function', - 'it directly', since='6.2') - - return self - - -class SelfCallDict(SelfCallMixin, dict): - - """Dict with SelfCallMixin. - - .. versionadded:: 3.0 - .. deprecated:: 6.2 - """ - - -class SelfCallString(SelfCallMixin, str): - - """String with SelfCallMixin. - - .. versionadded:: 3.0 - .. deprecated:: 6.2 - """ - - -def open_archive(filename: str, mode: str = 'rb', use_extension: bool = True): - """ - Open a file and uncompress it if needed. - - This function supports bzip2, gzip, 7zip, lzma, and xz as compression - containers. It uses the packages available in the standard library for - bzip2, gzip, lzma, and xz so they are always available. 7zip is only - available when a 7za program is available and only supports reading - from it. - - The compression is either selected via the magic number or file ending. - - .. versionadded:: 3.0 - - :param filename: The filename. - :param use_extension: Use the file extension instead of the magic number - to determine the type of compression (default True). Must be True when - writing or appending. - :param mode: The mode in which the file should be opened. It may either be - 'r', 'rb', 'a', 'ab', 'w' or 'wb'. All modes open the file in binary - mode. It defaults to 'rb'. - :raises ValueError: When 7za is not available or the opening mode is - unknown or it tries to write a 7z archive. - :raises FileNotFoundError: When the filename doesn't exist and it tries - to read from it or it tries to determine the compression algorithm. - :raises OSError: When it's not a 7z archive but the file extension is 7z. - It is also raised by bz2 when its content is invalid. gzip does not - immediately raise that error but only on reading it. - :raises lzma.LZMAError: When error occurs during compression or - decompression or when initializing the state with lzma or xz. - :raises ImportError: When file is compressed with bz2 but neither bz2 nor - bz2file is importable, or when file is compressed with lzma or xz but - lzma is not importable. - :return: A file-like object returning the uncompressed data in binary mode. - :rtype: file-like object - """ - # extension_map maps magic_number to extension. - # Unfortunately, legacy LZMA container has no magic number - extension_map = { - b'BZh': 'bz2', - b'\x1F\x8B\x08': 'gz', - b"7z\xBC\xAF'\x1C": '7z', - b'\xFD7zXZ\x00': 'xz', - } - - if mode in ('r', 'a', 'w'): - mode += 'b' - elif mode not in ('rb', 'ab', 'wb'): - raise ValueError('Invalid mode: "{}"'.format(mode)) - - if use_extension: - # if '.' not in filename, it'll be 1 character long but otherwise - # contain the period - extension = filename[filename.rfind('.'):][1:] - else: - if mode != 'rb': - raise ValueError('Magic number detection only when reading') - with open(filename, 'rb') as f: - magic_number = f.read(8) - - for pattern in extension_map: - if magic_number.startswith(pattern): - extension = extension_map[pattern] - break - else: - extension = '' - - if extension == 'bz2': - if isinstance(bz2, ImportError): - raise bz2 - binary = bz2.BZ2File(filename, mode) - - elif extension == 'gz': - binary = gzip.open(filename, mode) - - elif extension == '7z': - if mode != 'rb': - raise NotImplementedError('It is not possible to write a 7z file.') - - try: - process = subprocess.Popen(['7za', 'e', '-bd', '-so', filename], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=65535) - except OSError: - raise ValueError('7za is not installed or cannot uncompress "{}"' - .format(filename)) - - stderr = process.stderr.read() - process.stderr.close() - if stderr != b'': - process.stdout.close() - raise OSError( - 'Unexpected STDERR output from 7za {}'.format(stderr)) - binary = process.stdout - - elif extension in ('lzma', 'xz'): - if isinstance(lzma, ImportError): - raise lzma - lzma_fmts = {'lzma': lzma.FORMAT_ALONE, 'xz': lzma.FORMAT_XZ} - binary = lzma.open(filename, mode, format=lzma_fmts[extension]) - - else: # assume it's an uncompressed file - binary = open(filename, 'rb') - - return binary - - -def merge_unique_dicts(*args, **kwargs): - """ - Return a merged dict and make sure that the original dicts keys are unique. - - The positional arguments are the dictionaries to be merged. It is also - possible to define an additional dict using the keyword arguments. - - .. versionadded: 3.0 - """ - args = list(args) + [dict(kwargs)] - conflicts = set() - result = {} - for arg in args: - conflicts |= set(arg.keys()) & set(result.keys()) - result.update(arg) - if conflicts: - raise ValueError('Multiple dicts contain the same keys: {}' - .format(', '.join(sorted(str(key) - for key in conflicts)))) - return result - - -def file_mode_checker( - filename: str, - mode: int = 0o600, - quiet: bool = False, - create: bool = False -): - """Check file mode and update it, if needed. - - .. versionadded: 3.0 - - :param filename: filename path - :param mode: requested file mode - :param quiet: warn about file mode change if False. - :param create: create the file if it does not exist already - :raise IOError: The file does not exist and `create` is False. - """ - try: - st_mode = os.stat(filename).st_mode - except OSError: # file does not exist - if not create: - raise - os.close(os.open(filename, os.O_CREAT | os.O_EXCL, mode)) - return - - warn_str = 'File {0} had {1:o} mode; converted to {2:o} mode.' - if stat.S_ISREG(st_mode) and (st_mode - stat.S_IFREG != mode): - os.chmod(filename, mode) - # re-read and check changes - if os.stat(filename).st_mode != st_mode and not quiet: - warn(warn_str.format(filename, st_mode - stat.S_IFREG, mode)) - - -def compute_file_hash(filename: str, sha: str = 'sha1', bytes_to_read=None): - """Compute file hash. - - Result is expressed as hexdigest(). - - .. versionadded: 3.0 - - :param filename: filename path - :param sha: hashing function among the following in hashlib: - md5(), sha1(), sha224(), sha256(), sha384(), and sha512() - function name shall be passed as string, e.g. 'sha1'. - :param bytes_to_read: only the first bytes_to_read will be considered; - if file size is smaller, the whole file will be considered. - :type bytes_to_read: None or int - - """ - size = os.path.getsize(filename) - if bytes_to_read is None: - bytes_to_read = size - else: - bytes_to_read = min(bytes_to_read, size) - step = 1 << 20 - - shas = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'] - assert sha in shas - sha = getattr(hashlib, sha)() # sha instance - - with open(filename, 'rb') as f: - while bytes_to_read > 0: - read_bytes = f.read(min(bytes_to_read, step)) - assert read_bytes # make sure we actually read bytes - bytes_to_read -= len(read_bytes) - sha.update(read_bytes) - return sha.hexdigest() - - -def cached(*arg: Callable) -> Any: - """Decorator to cache information of an object. - - The wrapper adds an attribute to the instance which holds the result - of the decorated method. The attribute's name is the method name - with preleading underscore. - - Usage:: - - @cached - def this_method(self) - - @cached - def that_method(self, force=False) - - No parameter may be used with this decorator. Only a force parameter - may be used with the decorated method. All other parameters are - discarded and lead to a TypeError. - - .. note:: A property must be decorated on top of the property method - below other decorators. This decorator must not be used with - functions. - .. versionadded:: 7.3 - - :raises TypeError: decorator must be used without arguments - """ - fn = arg and arg[0] - if not callable(fn): - raise TypeError( - '"cached" decorator must be used without arguments.') from None - - @wraps(fn) - def wrapper(obj: object, *, force=False) -> Any: - cache_name = '_' + fn.__name__ - if force: - with suppress(AttributeError): - delattr(obj, cache_name) - try: - return getattr(obj, cache_name) - except AttributeError: - val = fn(obj) - setattr(obj, cache_name, val) - return val - - return wrapper - - -# Deprecate objects which has to be imported from tools.collections instead -wrapper = ModuleDeprecationWrapper(__name__) -wrapper.add_deprecated_attr( - 'CombinedError', - replacement_name='pywikibot.tools.collections.CombinedError', - since='7.6.0') -wrapper.add_deprecated_attr( - 'DequeGenerator', - replacement_name='pywikibot.tools.collections.DequeGenerator', - since='7.6.0') -wrapper.add_deprecated_attr( - 'EmptyDefault', - replacement_name='pywikibot.tools.collections.EmptyDefault', - since='7.6.0') -wrapper.add_deprecated_attr( - 'SizedKeyCollection', - replacement_name='pywikibot.tools.collections.SizedKeyCollection', - since='7.6.0') -wrapper.add_deprecated_attr( - 'EMPTY_DEFAULT', - replacement_name='pywikibot.tools.collections.EMPTY_DEFAULT', - since='7.6.0') - -# Deprecate objects which has to be imported from tools.itertools instead -wrapper.add_deprecated_attr( - 'itergroup', - replacement_name='pywikibot.tools.itertools.itergroup', - since='7.6.0') -wrapper.add_deprecated_attr( - 'islice_with_ellipsis', - replacement_name='pywikibot.tools.itertools.islice_with_ellipsis', - since='7.6.0') -wrapper.add_deprecated_attr( - 'intersect_generators', - replacement_name='pywikibot.tools.itertools.intersect_generators', - since='7.6.0') -wrapper.add_deprecated_attr( - 'roundrobin_generators', - replacement_name='pywikibot.tools.itertools.roundrobin_generators', - since='7.6.0') -wrapper.add_deprecated_attr( - 'filter_unique', - replacement_name='pywikibot.tools.itertools.filter_unique', - since='7.6.0') diff --git a/pywikibot/userinterfaces/terminal_interface_base.py b/pywikibot/userinterfaces/terminal_interface_base.py index 3286355..a21d376 100644 --- a/pywikibot/userinterfaces/terminal_interface_base.py +++ b/pywikibot/userinterfaces/terminal_interface_base.py @@ -22,7 +22,8 @@ StandardOption, ) from pywikibot.logging import INFO, INPUT, STDOUT, VERBOSE, WARNING -from pywikibot.tools import issue_deprecation_warning, RLock +from pywikibot.tools import issue_deprecation_warning +from pywikibot.tools.threading import RLock from pywikibot.userinterfaces import transliteration from pywikibot.userinterfaces._interface_base import ABUIC diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 265e4ee..34944f7 100755 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -132,7 +132,7 @@ from pywikibot.pagegenerators import ( XMLDumpPageGenerator as _XMLDumpPageGenerator, ) -from pywikibot.tools import ThreadList +from pywikibot.tools.threading import ThreadList try: diff --git a/tests/__init__.py b/tests/__init__.py index e478711..f17f05a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -119,7 +119,6 @@ 'tests', 'textlib', 'thanks', - 'thread', 'time', 'timestripper', 'token', @@ -127,6 +126,7 @@ 'tools_chars', 'tools_deprecate', 'tools_formatter', + 'tools_threading', 'ui', 'ui_options', 'upload', diff --git a/tests/thread_tests.py b/tests/tools_threading_tests.py similarity index 94% rename from tests/thread_tests.py rename to tests/tools_threading_tests.py index 61c313b..439a4d7 100755 --- a/tests/thread_tests.py +++ b/tests/tools_threading_tests.py @@ -8,7 +8,7 @@ import unittest from contextlib import suppress -from pywikibot.tools import ThreadedGenerator +from pywikibot.tools.threading import ThreadedGenerator from tests.aspects import TestCase -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/834652 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I493f5362670f57326ceef38e43ca172179d1eef0 Gerrit-Change-Number: 834652 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged