Pywikibot-commits

pywikibot-commits@lists.wikimedia.org

1 participants
13480 discussions

[Gerrit] ...core[master]: [IMPR] improvements for CategoryRedirectBot
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/979900?usp=email ) Change subject: [IMPR] improvements for CategoryRedirectBot ...................................................................... [IMPR] improvements for CategoryRedirectBot - derive CategoryRedirectBot from AutomaticTWSummaryBot - L10N detection during CategoryRedirectBot.__init__ - move loading record data into load_record() method - enable edit_requests log if moving categories fails - use check_hard_redirect as treat method - move code from run() method to check_soft_redirect and use it as treat method - add touch method for null edits - add methods to setup hard and soft redirects treats - change run method to only prepare BaseBot.run for different treats - only save log if necessary Bug: T276345 Bug: T351951 Bug: T352970 Change-Id: I27445dfe11f47f5867a64a20a528c28d79dfb0dc --- M scripts/category_redirect.py 1 file changed, 369 insertions(+), 241 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/scripts/category_redirect.py b/scripts/category_redirect.py index c8f1977..897bedd 100755 --- a/scripts/category_redirect.py +++ b/scripts/category_redirect.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 """This bot will move pages out of redirected categories. -The bot will look for categories that are marked with a category redirect -template, take the first parameter of the template as the target of the -redirect, and move all pages and subcategories of the category there. It -also changes hard redirects into soft redirects, and fixes double redirects. -A log is written under <userpage>/category_redirect_log. Only category pages -that haven't been edited for a certain cooldown period (currently 7 days) -are taken into account. +The bot will look for categories that are marked with a category +redirect template, take the first parameter of the template as the +target of the redirect, and move all pages and subcategories of the +category there. It also changes hard redirects into soft redirects, and +fixes double redirects. A log is written under +``<userpage>/category_redirect_log``. A log is written under +``<userpage>/category_edit_requests`` if a page cannot be moved to be +done manually. Only category pages that haven't been edited for a +certain cooldown period (default 7 days) are taken into account. The following parameters are supported: @@ -45,8 +47,15 @@ import pywikibot from pywikibot import config, i18n, pagegenerators from pywikibot.backports import removeprefix -from pywikibot.bot import ConfigParserBot, SingleSiteBot -from pywikibot.exceptions import CircularRedirectError, Error, NoPageError +from pywikibot.bot import AutomaticTWSummaryBot, ConfigParserBot, SingleSiteBot +from pywikibot.exceptions import ( + CircularRedirectError, + Error, + LockedPageError, + NoCreateError, + NoPageError, + PageSaveRelatedError, +) LOG_SIZE = 7 # Number of items to keep in active log @@ -54,14 +63,26 @@ CAT_REDIRECT_CAT = 'Q4616723' # Category that contains non-empty redirected category pages TINY_CAT_REDIRECT_CAT = 'Q8099903' +MOVE_COMMENT = 'category_redirect-change-category' +REDIR_COMMENT = 'category_redirect-add-template' +DBL_REDIR_COMMENT = 'category_redirect-fix-double' +MAINT_COMMENT = 'category_redirect-comment' -class CategoryRedirectBot(ConfigParserBot, SingleSiteBot): +class CategoryRedirectBot( + ConfigParserBot, + SingleSiteBot, + AutomaticTWSummaryBot +): """Page category update bot. .. versionchanged:: 7.0 CategoryRedirectBot is a ConfigParserBot + + .. versionchanged:: 9.0 + A logentry is writen to <userpage>/category_edit_requests if a + page cannot be moved """ update_options = { @@ -76,25 +97,30 @@ self.log_text = [] self.edit_requests = [] self.problems = [] - self.template_list = [] - self.cat = None - self.log_page = pywikibot.Page(self.site, - 'User:{}/category redirect log' - .format(self.site.username())) - self.move_comment = 'category_redirect-change-category' - self.redir_comment = 'category_redirect-add-template' - self.dbl_redir_comment = 'category_redirect-fix-double' - self.maint_comment = 'category_redirect-comment' + self.record = {} + self.newredirs = [] + self.oldstart = None + + self.log_page = pywikibot.Page( + self.site, f'User:{self.site.username()}/category redirect log') + self.edit_request_text = i18n.twtranslate( self.site, 'category_redirect-edit-request') + '\n~~~~' self.edit_request_item = i18n.twtranslate( self.site, 'category_redirect-edit-request-item') + # validate L10N + self.cat = self.get_cat() + if not self.cat: + raise Error(f'No redirect category found for {self.site}') + self.template_list = self.site.category_redirects() + if not self.template_list: + raise Error(f'No redirect templates defined for {self.site}') + def get_cat(self): """Specify the category page.""" item = TINY_CAT_REDIRECT_CAT if self.opt.tiny else CAT_REDIRECT_CAT - self.cat = self.site.page_from_repository(item) - return self.cat is not None + return self.site.page_from_repository(item) def move_contents(self, old_cat_title: str, new_cat_title: str, edit_summary: str) -> tuple[int, int]: @@ -112,9 +138,16 @@ # Move articles found, moved = 0, 0 - for article in old_cat.members(): - found += 1 - moved += article.change_category(old_cat, new_cat, summary=summary) + for found, article in enumerate(old_cat.members(), start=1): + done = article.change_category(old_cat, new_cat, summary=summary) + if done: + moved += 1 + else: + self.edit_requests.append({ + 'title': article.title(asLink=True, textlink=True), + 'oldcat': old_cat.title(asLink=True, textlink=True), + 'newcat': new_cat.title(asLink=True, textlink=True)} + ) if article.namespace() != 10: continue @@ -125,13 +158,22 @@ try: doc.get() except Error: - pass + continue + + done = doc.change_category(old_cat, new_cat, summary=summary) + if done: + moved += 1 else: - moved += doc.change_category(old_cat, new_cat, - summary=summary) + self.edit_requests.append({ + 'title': doc.title(asLink=True, textlink=True), + 'oldcat': old_cat.title(asLink=True, textlink=True), + 'newcat': new_cat.title(asLink=True, textlink=True)} + ) if found: + self.counter['move'] += moved pywikibot.info(f'{old_cat}: {found} found, {moved} moved') + return found, moved def ready_to_edit(self, cat): @@ -146,16 +188,20 @@ log_text = self.log_page.get() except NoPageError: log_text = '' + log_items = {} header = None + for line in log_text.splitlines(): if line.startswith('==') and line.endswith('=='): header = line[2:-2].strip() if header is not None: log_items.setdefault(header, []) log_items[header].append(line) + if len(log_items) < LOG_SIZE: return log_text + # sort by keys and keep the first (LOG_SIZE-1) values keep = [text for (key, text) in sorted(log_items.items(), reverse=True)[:LOG_SIZE - 1]] @@ -172,96 +218,105 @@ log_text += ('\n\n' + message) return log_text + def setup_hard_redirect(self): + """Setup hard redirect task.""" + pywikibot.info('Checking hard-redirect category pages.') + self.summary_key = REDIR_COMMENT + self.generator = self.site.allpages( + namespace=14, filterredir=True, content=True) + self.treat_page = self.check_hard_redirect + def check_hard_redirect(self) -> None: - """ - Check for hard-redirected categories. + """Check for hard-redirected categories. Check categories that are not already marked with an appropriate - softredirect template. + softredirect template and replace the content with a redirect + template. """ - pywikibot.info('Checking hard-redirect category pages.') - comment = i18n.twtranslate(self.site, self.redir_comment) - - # generator yields all hard redirect pages in namespace 14 - for page in self.site.allpages(namespace=14, filterredir=True, - content=True): - if page.isCategoryRedirect(): - # this is already a soft-redirect, so skip it (for now) - continue - try: - target = page.getRedirectTarget() - except CircularRedirectError: - target = page - message = i18n.twtranslate( - self.site, 'category_redirect-problem-self-linked', - {'oldcat': page.title(as_link=True, textlink=True)}) - self.problems.append(message) - except RuntimeError: - # race condition: someone else removed the redirect while we - # were checking for it - continue - - if not target.is_categorypage(): - message = i18n.twtranslate( - self.site, 'category_redirect-problem-hard', { - 'oldcat': page.title(as_link=True, textlink=True), - 'page': target.title(as_link=True, textlink=True) - }) - self.problems.append(message) - continue - - # this is a hard-redirect to a category page - newtext = ('{{%(template)s|%(cat)s}}' - % {'cat': target.title(with_ns=False), - 'template': self.template_list[0]}) - params = { - 'ns': self.site.namespaces.TEMPLATE.custom_prefix(), - 'template': self.template_list[0], - 'oldcat': page.title(as_link=True, textlink=True) - } - try: - page.text = newtext - page.save(comment) - message = i18n.twtranslate( - self.site, 'category_redirect-log-added', params) - self.log_text.append(message) - except Error as e: - pywikibot.error(e) - message = i18n.twtranslate( - self.site, 'category_redirect-log-add-failed', params) - self.log_text.append(message) - - def run(self) -> None: - """Run the bot.""" - # validate L10N - self.template_list = self.site.category_redirects() - if not self.template_list: - pywikibot.warning(f'No redirect templates defined for {self.site}') - return - if not self.get_cat(): - pywikibot.warning(f'No redirect category found for {self.site}') + page = self.current_page + if page.isCategoryRedirect(): + # this is already a soft-redirect, so skip it (for now) return - self.user = self.site.user() # invokes login() - self.newredirs = [] + try: + target = page.getRedirectTarget() + except CircularRedirectError: + target = page + message = i18n.twtranslate( + self.site, 'category_redirect-problem-self-linked', + {'oldcat': page.title(as_link=True, textlink=True)}) + self.problems.append(message) + except RuntimeError: + # race condition: someone else removed the redirect while we + # were checking for it + return - localtime = time.localtime() - today = '{:04d}-{:02d}-{:02d}'.format(*localtime[:3]) + if not target.is_categorypage(): + message = i18n.twtranslate( + self.site, 'category_redirect-problem-hard', { + 'oldcat': page.title(as_link=True, textlink=True), + 'page': target.title(as_link=True, textlink=True) + }) + self.problems.append(message) + return + + # this is a hard-redirect to a category page + newtext = ('{{%(template)s|%(cat)s}}' + % {'cat': target.title(with_ns=False), + 'template': self.template_list[0]}) + params = { + 'ns': self.site.namespaces.TEMPLATE.custom_prefix(), + 'template': self.template_list[0], + 'oldcat': page.title(as_link=True, textlink=True) + } + + try: + self.put_current(newtext) + message = i18n.twtranslate( + self.site, 'category_redirect-log-added', params) + except Error as e: + pywikibot.error(e) + message = i18n.twtranslate( + self.site, 'category_redirect-log-add-failed', params) + + self.log_text.append(message) + + def load_record(self) -> None: + """Load record from data file and create a backup file.""" self.datafile = pywikibot.config.datafilepath( f'{self.site.dbName()}-catmovebot-data') - try: - with open(self.datafile, 'rb') as inp: - self.record = pickle.load(inp) - except OSError: - self.record = {} + with suppress(OSError), open(self.datafile, 'rb') as inp: + self.record = pickle.load(inp) if self.record: with open(self.datafile + '.bak', 'wb') as f: pickle.dump(self.record, f, protocol=config.pickle_protocol) + + def touch(self, page) -> None: + """Touch the given page.""" + try: + page.touch() + except (NoCreateError, NoPageError): + pywikibot.error(f'Page {page.title(as_link=True)} does not exist.') + except LockedPageError: + pywikibot.error(f'Page {page.title(as_link=True)} is locked.') + except PageSaveRelatedError as e: + pywikibot.error(f'Page {page} not saved:\n{e.args}') + else: + self.counter['touch'] += 1 + + def setup_soft_redirect(self): + """Setup soft redirect task.""" + pywikibot.info(f'\nChecking {self.cat.categoryinfo["subcats"]}' + ' category redirect pages') + self.load_record() + localtime = time.localtime() + self.today = '{:04d}-{:02d}-{:02d}'.format(*localtime[:3]) + # regex to match soft category redirects # TODO: enhance and use textlib.MultiTemplateMatchBuilder # note that any templates containing optional "category:" are # incorrect and will be fixed by the bot - template_regex = re.compile( + self.template_regex = re.compile( r"""{{{{\s*(?:{prefix}\s*:\s*)? # optional "template:" (?:{template})\s*\| # catredir template name (\s*{catns}\s*:\s*)? # optional "category:" @@ -273,42 +328,55 @@ catns=self.site.namespace(14)), re.I | re.X) - self.check_hard_redirect() - - comment = i18n.twtranslate(self.site, self.move_comment) - counts = {} nonemptypages = [] - redircat = self.cat - - pywikibot.info('\nChecking {} category redirect pages' - .format(redircat.categoryinfo['subcats'])) catpages = set() - for cat in redircat.subcategories(): - catpages.add(cat) - cat_title = cat.title(with_ns=False) - if 'category redirect' in cat_title: - message = i18n.twtranslate( - self.site, 'category_redirect-log-ignoring', - {'oldcat': cat.title(as_link=True, textlink=True)}) - self.log_text.append(message) - continue - if hasattr(cat, '_catinfo'): - # skip empty categories that don't return a "categoryinfo" key - catdata = cat.categoryinfo - if 'size' in catdata and int(catdata['size']): - # save those categories that have contents - nonemptypages.append(cat) - if cat_title not in self.record: - # make sure every redirect has a self.record entry - self.record[cat_title] = {today: None} - with suppress(Error): - self.newredirs.append('*# {} → {}'.format( - cat.title(as_link=True, textlink=True), - cat.getCategoryRedirectTarget().title( - as_link=True, textlink=True))) - # do a null edit on cat - with suppress(Exception): - cat.save() + + # No throttle for touch edits + save_throttle = config.put_throttle + config.put_throttle = 0 + + do_exit = False + try: + for cat in self.cat.subcategories(): + if do_exit: + break + + self.counter['read'] += 1 + cat_title = cat.title(with_ns=False) + if 'category redirect' in cat_title: + message = i18n.twtranslate( + self.site, 'category_redirect-log-ignoring', + {'oldcat': cat.title(as_link=True, textlink=True)}) + self.log_text.append(message) + continue + + if hasattr(cat, '_catinfo'): + # skip empty categories that don't return a "categoryinfo" + # key + catdata = cat.categoryinfo + if 'size' in catdata and int(catdata['size']): + # save those categories that have contents + nonemptypages.append(cat) + + if cat_title not in self.record: + # make sure every redirect has a self.record entry + self.record[cat_title] = {self.today: None} + with suppress(Error): + self.newredirs.append('*# {} → {}'.format( + cat.title(as_link=True, textlink=True), + cat.getCategoryRedirectTarget().title( + as_link=True, textlink=True))) + + # do a null edit on cat + if cat not in catpages: + self.touch(cat) + + catpages.add(cat) + except KeyboardInterrupt: + pywikibot.info('KeyboardInterrupt during subcategory checks...') + do_exit = True + + config.put_throttle = save_throttle # delete self.record entries for non-existent categories for cat_name in list(self.record): @@ -316,126 +384,158 @@ self.catprefix + cat_name) not in catpages: del self.record[cat_name] - pywikibot.info('\nMoving pages out of {} redirected categories.' - .format(len(nonemptypages))) + pywikibot.info(f'\nMoving pages out of {len(nonemptypages)}' + ' redirected categories.') + self.summary_key = DBL_REDIR_COMMENT + self.generator = pagegenerators.PreloadingGenerator(nonemptypages) + self.treat_page = self.check_soft_redirect - for cat in pagegenerators.PreloadingGenerator(nonemptypages): - i18n_param = {'oldcat': cat.title(as_link=True, textlink=True)} + def check_soft_redirect(self) -> None: + """Check for soft-redirected categories.""" + cat = self.current_page + i18n_param = {'oldcat': cat.title(as_link=True, textlink=True)} - try: - if not cat.isCategoryRedirect(): - message = i18n.twtranslate( - self.site, - 'category_redirect-log-false-positive', - i18n_param - ) - self.log_text.append(message) - continue - except Error: + try: + if not cat.isCategoryRedirect(): + message = i18n.twtranslate( + self.site, + 'category_redirect-log-false-positive', + i18n_param + ) + self.log_text.append(message) + return + except Error: + message = i18n.twtranslate(self.site, + 'category_redirect-log-not-loaded', + i18n_param) + self.log_text.append(message) + return + + cat_title = cat.title(with_ns=False) + if not self.ready_to_edit(cat): + message = i18n.twtranslate(self.site, + 'category_redirect-log-skipping', + i18n_param) + self.log_text.append(message) + return + + dest = cat.getCategoryRedirectTarget() + if not dest.exists(): + message = i18n.twtranslate( + self.site, 'category_redirect-problem-redirects', { + 'oldcat': cat.title(as_link=True, textlink=True), + 'redpage': dest.title(as_link=True, textlink=True) + }) + self.problems.append(message) + # do a null edit on cat to update any special redirect + # categories this wiki might maintain + self.touch(cat) + return + + if dest.isCategoryRedirect(): + double = dest.getCategoryRedirectTarget() + if double in (dest, cat): message = i18n.twtranslate(self.site, - 'category_redirect-log-not-loaded', + 'category_redirect-log-loop', i18n_param) self.log_text.append(message) - continue - - cat_title = cat.title(with_ns=False) - if not self.ready_to_edit(cat): - counts[cat_title] = None - message = i18n.twtranslate(self.site, - 'category_redirect-log-skipping', - i18n_param) - self.log_text.append(message) - continue - - dest = cat.getCategoryRedirectTarget() - if not dest.exists(): + # do a null edit on cat + self.touch(cat) + else: message = i18n.twtranslate( - self.site, 'category_redirect-problem-redirects', { + self.site, 'category_redirect-log-double', { 'oldcat': cat.title(as_link=True, textlink=True), - 'redpage': dest.title(as_link=True, textlink=True) + 'newcat': dest.title(as_link=True, textlink=True), + 'targetcat': double.title( + as_link=True, textlink=True) }) - self.problems.append(message) - # do a null edit on cat to update any special redirect - # categories this wiki might maintain - with suppress(Exception): - cat.save() - continue - - if dest.isCategoryRedirect(): - double = dest.getCategoryRedirectTarget() - if double in (dest, cat): - message = i18n.twtranslate(self.site, - 'category_redirect-log-loop', - i18n_param) - self.log_text.append(message) - # do a null edit on cat - with suppress(Exception): - cat.save() - else: + self.log_text.append(message) + oldtext = cat.text + # remove the old redirect from the old text, + # leaving behind any non-redirect text + oldtext = self.template_regex.sub('', oldtext) + newtext = ('{{%(redirtemp)s|%(ncat)s}}' + % {'redirtemp': self.template_list[0], + 'ncat': double.title(with_ns=False)}) + newtext += oldtext.strip() + try: + self.put_current(newtext) + except Error as e: message = i18n.twtranslate( - self.site, 'category_redirect-log-double', { - 'oldcat': cat.title(as_link=True, textlink=True), - 'newcat': dest.title(as_link=True, textlink=True), - 'targetcat': double.title( - as_link=True, textlink=True) - }) + self.site, 'category_redirect-log-failed', + {'error': e}) self.log_text.append(message) - oldtext = cat.text - # remove the old redirect from the old text, - # leaving behind any non-redirect text - oldtext = template_regex.sub('', oldtext) - newtext = ('{{%(redirtemp)s|%(ncat)s}}' - % {'redirtemp': self.template_list[0], - 'ncat': double.title(with_ns=False)}) - newtext += oldtext.strip() - try: - cat.text = newtext - cat.save(i18n.twtranslate(self.site, - self.dbl_redir_comment)) - except Error as e: - message = i18n.twtranslate( - self.site, 'category_redirect-log-failed', - {'error': e}) - self.log_text.append(message) - continue + return - found, moved = self.move_contents( - cat_title, dest.title(with_ns=False), comment) - if found: - self.record[cat_title][today] = found - message = i18n.twtranslate( - self.site, 'category_redirect-log-moved', { - 'oldcat': cat.title(as_link=True, textlink=True), - 'found': found, - 'moved': moved - }) - self.log_text.append(message) - counts[cat_title] = found - # do a null edit on cat - with suppress(Exception): - cat.save() + found, moved = self.move_contents( + cat_title, + dest.title(with_ns=False), + i18n.twtranslate(self.site, MOVE_COMMENT) + ) - self.teardown() + if found: + self.record[cat_title][self.today] = found + message = i18n.twtranslate( + self.site, 'category_redirect-log-moved', { + 'oldcat': cat.title(as_link=True, textlink=True), + 'found': found, + 'moved': moved + }) + self.log_text.append(message) + + # do a null edit on cat + self.touch(cat) + + def run(self) -> None: + """Run the bot.""" + self.user = self.site.user() # invokes login() + + # process hard category redirects + oldexit = self.exit + self.exit = lambda: None + self.setup = self.setup_hard_redirect + super().run() + + # save timestamp and prepare the next step + self.oldstart = self._start_ts + if not self.generator_completed \ + and (self.opt.always + or not pywikibot.input_yn( + 'Continue with soft category redirects', + automatic_quit=False)): + oldexit() + return + + # process soft category redirects + self.exit = oldexit + self.generator_completed = False + self.setup = self.setup_soft_redirect + super().run() def teardown(self) -> None: """Write self.record to file and save logs.""" - with open(self.datafile, 'wb') as f: - pickle.dump(self.record, f, protocol=config.pickle_protocol) + self._start_ts = self.oldstart + if self.record: + with open(self.datafile, 'wb') as f: + pickle.dump(self.record, f, protocol=config.pickle_protocol) - self.log_text.sort() - self.problems.sort() - self.newredirs.sort() - comment = i18n.twtranslate(self.site, self.maint_comment) - message = i18n.twtranslate(self.site, 'category_redirect-log-new') - date_line = '\n== {}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}Z ==\n' \ - .format(*time.gmtime()[:6]) - self.log_page.text = (date_line - + '\n'.join(self.log_text) - + '\n* ' + message + '\n' - + '\n'.join(self.newredirs) - + '\n' + '\n'.join(self.problems) - + '\n' + self.get_log_text()) - self.log_page.save(comment) + comment = i18n.twtranslate(self.site, MAINT_COMMENT) + + if self.log_text or self.problems or self.newredirs: + self.log_text.sort() + self.problems.sort() + self.newredirs.sort() + message = i18n.twtranslate(self.site, 'category_redirect-log-new') + date_line = '\n== {}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}Z ==\n' \ + .format(*time.gmtime()[:6]) + self.log_page.text = (date_line + + '\n'.join(self.log_text) + + '\n* ' + message + '\n' + + '\n'.join(self.newredirs) + + '\n' + '\n'.join(self.problems) + + '\n' + self.get_log_text()) + self.log_page.save(comment) + if self.edit_requests: edit_request_page = pywikibot.Page( self.site, f'User:{self.user}/category edit requests') @@ -461,8 +561,12 @@ else: # generic handling of we have boolean options options[arg[1:]] = True - bot = CategoryRedirectBot(**options) - bot.run() + try: + bot = CategoryRedirectBot(**options) + except Error as e: + pywikibot.bot.suggest_help(exception=e) + else: + bot.run() if __name__ == '__main__': -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/979900?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I27445dfe11f47f5867a64a20a528c28d79dfb0dc Gerrit-Change-Number: 979900 Gerrit-PatchSet: 17 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Aram <arambakr1620(a)gmail.com> Gerrit-Reviewer: D3r1ck01 <dalangi-ctr(a)wikimedia.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-CC: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [bugfix] remove content parameter of ItemPage.page_gen method
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007387?usp=email ) Change subject: [bugfix] remove content parameter of ItemPage.page_gen method ...................................................................... [bugfix] remove content parameter of ItemPage.page_gen method The filter needs the page.text and the text is preloaded Page for page already. Now always use the site.preload method to bulk load the content. Bug: T358635 Change-Id: I62dca5845492a636cf556761d95bde5f5210d9f8 --- M pywikibot/proofreadpage.py 1 file changed, 22 insertions(+), 10 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py index 2df5eb1..e5dc94d 100644 --- a/pywikibot/proofreadpage.py +++ b/pywikibot/proofreadpage.py @@ -47,7 +47,7 @@ from pywikibot.data.api import ListGenerator, Request from pywikibot.exceptions import Error, InvalidTitleError, OtherPageSaveError from pywikibot.page import PageSourceType -from pywikibot.tools import MediaWikiVersion, cached +from pywikibot.tools import MediaWikiVersion, cached, remove_last_args try: @@ -1305,12 +1305,13 @@ """ return len(self._page_from_numbers) - def page_gen(self, start: int = 1, - end: int | None = None, - filter_ql: Sequence[int] | None = None, - only_existing: bool = False, - content: bool = True - ) -> Iterable[pywikibot.page.Page]: + @remove_last_args(['content']) # since 9.0.0 + def page_gen( + self, start: int = 1, + end: int | None = None, + filter_ql: Sequence[int] | None = None, + only_existing: bool = False + ) -> Iterable[pywikibot.page.Page]: """Return a page generator which yields pages contained in Index page. Range is [start ... end], extremes included. @@ -1324,7 +1325,6 @@ :param filter_ql: filters quality levels if None: all but 'Without Text'. :param only_existing: yields only existing pages. - :param content: preload content. """ if end is None: end = self.num_pages @@ -1346,8 +1346,7 @@ gen = [(self.get_number(p), p) for p in gen] gen = [p for n, p in sorted(gen)] - if content: - gen = self.site.preloadpages(gen) + gen = self.site.preloadpages(gen) # Filter by QL. gen = (p for p in gen if p.ql in filter_ql) # Yield only existing. -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007387?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I62dca5845492a636cf556761d95bde5f5210d9f8 Gerrit-Change-Number: 1007387 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [bugfix] Use site.code instead of site.lang in interwiki.py
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1005523?usp=email ) Change subject: [bugfix] Use site.code instead of site.lang in interwiki.py ...................................................................... [bugfix] Use site.code instead of site.lang in interwiki.py The site.code specifies the site. This may be different from language spezifier 'lang'. Therefore Use site.code instead of site.lang. Also add a new method get_alternative() to ask for a alternative page hint to prevent code duplication. Change-Id: I15d66b29d13bbf60c31c5a1cb0003d53601110ae --- M scripts/interwiki.py 1 file changed, 42 insertions(+), 21 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/scripts/interwiki.py b/scripts/interwiki.py index ead72d0..33102ef 100755 --- a/scripts/interwiki.py +++ b/scripts/interwiki.py @@ -821,6 +821,21 @@ or self.namespaceMismatch(page, target, counter) \ or self.wiktionaryMismatch(target) + def get_alternative( + self, + site: pywikibot.site.BaseSite + ) -> pywikibot.Page | None: + """Ask for an alternative Page for a given site. + + :param site: a BaseSite + """ + title = pywikibot.input(f'Give the alternative page for code ' + f'{site.code!r} (without site code)') + if title: + return pywikibot.Page(site, title) + + return None + def namespaceMismatch(self, linkingPage, linkedPage, counter) -> bool: """ Check whether or not the given page has a different namespace. @@ -836,9 +851,9 @@ # Allow for a mapping between different namespaces crossFrom = self.origin.site.family.crossnamespace.get( self.origin.namespace(), {}) - crossTo = crossFrom.get(self.origin.site.lang, + crossTo = crossFrom.get(self.origin.site.code, crossFrom.get('_default', {})) - nsmatch = crossTo.get(linkedPage.site.lang, + nsmatch = crossTo.get(linkedPage.site.code, crossTo.get('_default', [])) if linkedPage.namespace() in nsmatch: return False @@ -879,15 +894,10 @@ if choice == 'g': self.makeForcedStop(counter) elif choice == 'a': - newHint = pywikibot.input( - 'Give the alternative for language {}, not ' - 'using a language code:' - .format(linkedPage.site.lang)) - if newHint: - alternativePage = pywikibot.Page( - linkedPage.site, newHint) + alternative_page = self.get_alternative(linkedPage.site) + if alternative_page: # add the page that was entered by the user - self.addIfNew(alternativePage, counter, None) + self.addIfNew(alternative_page, counter, None) else: pywikibot.info( f'NOTE: ignoring {linkedPage} and its interwiki links') @@ -986,11 +996,8 @@ return (True, None) if choice == 'a': - newHint = pywikibot.input( - f'Give the alternative for language {page.site.lang}, ' - f'not using a language code:') - alternativePage = pywikibot.Page(page.site, newHint) - return (True, alternativePage) + alternative_page = self.get_alternative(page.site) + return (True, alternative_page) if choice == 'g': self.makeForcedStop(counter) @@ -1001,7 +1008,7 @@ def isIgnored(self, page) -> bool: """Return True if pages is to be ignored.""" - if page.site.lang in self.conf.neverlink: + if page.site.code in self.conf.neverlink: pywikibot.info(f'Skipping link {page} to an ignored language') return True @@ -1183,7 +1190,7 @@ # Ignore the interwiki links. iw = () if self.conf.lacklanguage \ - and self.conf.lacklanguage in (link.site.lang for link in iw): + and self.conf.lacklanguage in (link.site.code for link in iw): iw = () self.workonme = False if len(iw) < self.conf.minlinks: @@ -1269,10 +1276,9 @@ if dictName is not None: if self.origin: pywikibot.warning( - '{}:{} relates to {}:{}, which is an ' - 'auto entry {}({})' - .format(self.origin.site.lang, self.origin, - page.site.lang, page, dictName, year)) + f'{self.origin.site.code}:{self.origin} relates ' + f'to {page.site.code}:{page}, which is an auto ' + f'entry {dictName}({year})') # Abort processing if the bot is running in autonomous mode if self.conf.autonomous: -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1005523?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I15d66b29d13bbf60c31c5a1cb0003d53601110ae Gerrit-Change-Number: 1005523 Gerrit-PatchSet: 2 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: D3r1ck01 <dalangi-ctr(a)wikimedia.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [cleanup] use self.origin attribute instead of property
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1005504?usp=email ) Change subject: [cleanup] use self.origin attribute instead of property ...................................................................... [cleanup] use self.origin attribute instead of property There is no reason tjo have properties here instead of an attribute Change-Id: I8ed6bed0dd7bc28a5855893b2b0c1d0d49f4712a --- M pywikibot/interwiki_graph.py 1 file changed, 12 insertions(+), 10 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/interwiki_graph.py b/pywikibot/interwiki_graph.py index 66eb8be..40d67c4 100644 --- a/pywikibot/interwiki_graph.py +++ b/pywikibot/interwiki_graph.py @@ -62,7 +62,7 @@ :param origin: the page on the 'origin' wiki """ # Remember the "origin page" - self._origin = origin + self.origin = origin # found_in is a dictionary where pages are keys and lists of # pages are values. It stores where we found each page. @@ -72,15 +72,6 @@ if origin: self.found_in = {origin: []} - @property - def origin(self) -> pywikibot.page.Page | None: - """Page on the origin wiki.""" - return self._origin - - @origin.setter - def origin(self, value: pywikibot.page.Page | None) -> None: - self._origin = value - class GraphDrawer: -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1005504?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I8ed6bed0dd7bc28a5855893b2b0c1d0d49f4712a Gerrit-Change-Number: 1005504 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: Update MediaInfo editing tests
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1004301?usp=email ) Change subject: Update MediaInfo editing tests ...................................................................... Update MediaInfo editing tests Change MediaInfo edit tests so that they will login and write to Beta Commons instead of read-only testing. Bug: T355927 Change-Id: I2853b3191c92a511b3b868ed24223abef2a3df83 --- M pywikibot/families/commons_family.py M tests/file_tests.py 2 files changed, 119 insertions(+), 72 deletions(-) Approvals: jenkins-bot: Verified Matěj Suchánek: Looks good to me, but someone else must approve Xqt: Looks good to me, approved diff --git a/pywikibot/families/commons_family.py b/pywikibot/families/commons_family.py index 428dc16..30773bd 100644 --- a/pywikibot/families/commons_family.py +++ b/pywikibot/families/commons_family.py @@ -27,6 +27,9 @@ 'beta': 'commons.wikimedia.beta.wmflabs.org' } + # Sites we want to edit but not count as real languages + test_codes = ['test', 'beta'] + interwiki_forward = 'wikipedia' # Templates that indicate a category redirect diff --git a/tests/file_tests.py b/tests/file_tests.py index 40cc008..b5b999f 100755 --- a/tests/file_tests.py +++ b/tests/file_tests.py @@ -20,7 +20,6 @@ NoPageError, NoWikibaseEntityError, PageRelatedError, - UserRightsError, ) from tests import join_images_path from tests.aspects import TestCase @@ -456,112 +455,144 @@ item.statements, pywikibot.page._collections.ClaimCollection) -(a)unittest.expectedFailure # T355927 -class TestMediaInfoReadonlyEditing(TestCase): +class TestMediaInfoEditing(TestCase): """Test writing structured data of FilePage.""" - # Create valid API requests which will fail to missing permission. + login = True + write = True # commons.wikimedia.beta.wmflabs.org family = 'commons' code = 'beta' def test_edit_label(self): - """Test label editing without writing rights.""" - page = pywikibot.FilePage(self.site, 'File:123_4.jpg') - item = page.data_item() - error_message = 'User "None" does not have required user right "edit"' - - data = {} - lang = 'fi' - label = 'Edit label with editEntity and dict()' - data.update({'labels': {lang: label}}) - with self.assertRaisesRegex(UserRightsError, error_message): - item.editEntity(data, summary=label) - - data = item.get() - label = 'Edit label with editEntity and item.get()' - data.update({'labels': {lang: label}}) - with self.assertRaisesRegex(UserRightsError, error_message): - item.editEntity(data, summary=label) - - data = item.get(force=True) - label = 'Edit label with editEntity and item.get(force=True)' - data.update({'labels': {lang: label}}) - with self.assertRaisesRegex(UserRightsError, error_message): - item.editEntity(data, summary=label) - - lang = 'fi' - label = 'Edit label with editLabels and dict()' - data = {lang: label} - with self.assertRaisesRegex(UserRightsError, error_message): - item.editLabels(data, summary=label) - + """Test label editing.""" # Test label editing when file doesn't exists page = pywikibot.FilePage(self.site, 'File:123_4_DOESNT_EXISTS.jpg') item = page.data_item() + error_message = "Entity '-1' doesn't exist on commons:beta" + lang = 'fi' data = {} - lang = 'fi' label = 'Edit label of missing file with editEntity and dict()' data.update({'labels': {lang: label}}) - with self.assertRaises(UserRightsError): + with self.assertRaisesRegex(NoWikibaseEntityError, error_message): item.editEntity(data, summary=label) - lang = 'fi' label = 'Edit label of missing file with editLabels and dict()' data = {lang: label} - with self.assertRaisesRegex(UserRightsError, error_message): + with self.assertRaisesRegex(NoWikibaseEntityError, error_message): item.editLabels(data, summary=label) - def test_edit_claims(self): - """Test claim editing without writing rights.""" - error_message = 'User "None" does not have required user right "edit"' - wikidata_site = pywikibot.Site('wikidata', 'wikidata') - - # Test adding claim existing file + # Test label editing when file exists page = pywikibot.FilePage(self.site, 'File:123_4.jpg') item = page.data_item() + lang = 'fi' + + label = 'Edit label with editEntity and dict()' + data = {} + data.update({'labels': {lang: label}}) + item.editEntity(data, summary=label) + item.get(force=True) + self.assertEqual(item.labels['fi'], label) + + label = 'Edit label with editEntity and item' + item.labels['fi'] = label + item.editEntity(item.toJSON(), summary=label) + item.get(force=True) + self.assertEqual(item.labels['fi'], label) + + label = 'Edit label with editLabels and dict()' + data = {lang: label} + item.editLabels(data, summary=label) + item.get(force=True) + self.assertEqual(item.labels['fi'], label) + + def test_edit_claims(self): + """Test addClaim and removeClaim editing.""" + wikidata_site = pywikibot.Site('beta', 'wikidata') # Create claim - finna_id = 'test123' - new_claim = pywikibot.Claim(wikidata_site, 'P9478') - new_claim.setTarget(finna_id) - # Do actual edit - with self.assertRaisesRegex(UserRightsError, error_message): - item.addClaim(new_claim) - - # Test removing first claim using item.statements - for property_id in item.statements: - for statement in item.statements[property_id]: - with self.assertRaisesRegex(UserRightsError, error_message): - summary = f'Removing {property_id}' - item.removeClaims(statement, summary=summary) - break - - # Test removing first claim using item.claims - for property_id in item.claims: - for claim in item.claims[property_id]: - with self.assertRaisesRegex(UserRightsError, error_message): - summary = f'Removing {property_id}' - item.removeClaims(claim, summary=summary) - break + property_id = 'P766' # date of publication + new_claim = pywikibot.Claim(wikidata_site, 'P766') + new_claim_value = pywikibot.WbTime(2024, 1, 1) + new_claim.setTarget(new_claim_value) # Test adding claim to non-existing file page = pywikibot.FilePage(self.site, 'File:123_4_DOESNT_EXISTS.jpg') item = page.data_item() - # Create claim - finna_id = 'test123' - new_claim = pywikibot.Claim(wikidata_site, 'P9478') - new_claim.setTarget(finna_id) - - # Do actual edit + # Insert claim to non-existing file with self.assertRaises(NoWikibaseEntityError): item.addClaim(new_claim) + # Insert claim using site object to non-existing file + with self.assertRaises(NoWikibaseEntityError): + self.site.addClaim(item, new_claim) + + # Test adding claim existing file + page = pywikibot.FilePage(self.site, 'File:123_4.jpg') + item = page.data_item() + + # Test adding claim using MediaInfo.addClaim() + item.addClaim(new_claim) + + # Test that claim can be found + claim_found = False + item.get(force=True) + for statement in item.statements.get(property_id): + value = statement.getTarget() + if value == new_claim_value: + claim_found = True + summary = f'Removing {property_id} with {value}' + item.removeClaims(statement, summary=summary) + + self.assertTrue(claim_found) + + # Test that the claim was removed + + claim_found = False + item.get(force=True) + for statement in item.statements.get(property_id, []): + value = statement.getTarget() + if value == new_claim_value: + claim_found = True + + self.assertFalse(claim_found) + + # Add claim using site object + self.site.addClaim(item, new_claim) + + # Test that the claim can be found and remove it using site object + + claim_found = False + item.get(force=True) + for statement in item.statements.get(property_id): + value = statement.getTarget() + remove_statements = [] + if value == new_claim_value: + remove_statements.append(statement) + claim_found = True + + self.assertTrue(claim_found) + + # Note removeClaims() parameter needs to be array + summary = f'Removing {property_id} with {value} using site object' + self.site.removeClaims(remove_statements, summary=summary) + + # Test that the claims were actually removed + + claim_found = False + item.get(force=True) + for statement in item.statements.get(property_id, []): + value = statement.getTarget() + if value == new_claim_value: + claim_found = True + + self.assertFalse(claim_found) + if __name__ == '__main__': with suppress(SystemExit): -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1004301?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I2853b3191c92a511b3b868ed24223abef2a3df83 Gerrit-Change-Number: 1004301 Gerrit-PatchSet: 1 Gerrit-Owner: Zache-tool <kimmo.virtanen(a)gmail.com> Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: Populate MediaInfo._content with expected attributes when loaded
by jenkins-bot (Code Review) 02 Mar '24

02 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1006184?usp=email ) Change subject: Populate MediaInfo._content with expected attributes when loaded ...................................................................... Populate MediaInfo._content with expected attributes when loaded Added pageid, ns, title, lastrevid, modified, id values to _content attribute when it is loaded using mediainfo.get() so the format is identical with values returned by wbgetentities. Bug: T357608 Change-Id: I9178e3fe5a3a1ccba439864891bb1834fb28b050 --- M pywikibot/page/_wikibase.py M tests/file_tests.py 2 files changed, 56 insertions(+), 27 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/pywikibot/page/_wikibase.py b/pywikibot/page/_wikibase.py index b5de1b3..143e323 100644 --- a/pywikibot/page/_wikibase.py +++ b/pywikibot/page/_wikibase.py @@ -451,11 +451,6 @@ def get(self, force: bool = False) -> dict: """Fetch all MediaInfo entity data and cache it. - .. note:: This method may raise exception even if the associated file - exists because the mediainfo may not have been initialized yet. - :attr:`labels` and :attr:`statements` can still be accessed and - modified. :meth:`exists` suppresses the exception. - .. note:: dicts returned by this method are references to content of this entity and their modifying may indirectly cause unwanted change to the live content @@ -464,23 +459,51 @@ :raise NoWikibaseEntityError: if this entity doesn't exist :return: actual data which entity holds """ - if self.id == '-1': - if not force: - try: - data = self.file.latest_revision.slots['mediainfo']['*'] - except NoPageError as exc: + if force or not hasattr(self, '_content'): + if force: + self.file.clear_cache() + + # accessing latest_revision loads the file data + try: + latest_revision = self.file.latest_revision + except NoPageError as exc: + raise NoWikibaseEntityError(self) from exc + except Error as exc: + error_message = str(exc) + if 'is not a file' in error_message: raise NoWikibaseEntityError(self) from exc - except KeyError: - # reuse the reserved ID for better message - self.id = 'M' + str(self.file.pageid) - raise NoWikibaseEntityError(self) from None + else: + raise Error(self) from exc - self._content = jsonlib.loads(data) - self.id = self._content['id'] + # Create _content. Format is same as with wbgetentities + # https://commons.wikimedia.org/w/api.php?action=wbgetentities&ids=M20985340 + data = { + 'title': self.file.title, + 'lastrevid': latest_revision['revid'], + 'modified': str(latest_revision['timestamp']), + 'type': 'mediainfo', + 'pageid': self.file.pageid, + 'ns': self.file.namespace, + 'id': 'M' + str(self.file.pageid), + 'labels': {}, + 'statements': {} + } - self._assert_has_id() + # Update 'id', 'labels' and 'statements' if mediainfo is available. + # MediaInfo is returned only when it has values. + if 'mediainfo' in latest_revision.slots: + mediainfo_json = latest_revision.slots['mediainfo']['*'] + mediainfo_data = jsonlib.loads(mediainfo_json) + data.update(mediainfo_data) - return super().get(force=force) + self._content = data + self.id = self._content['id'] + + self._assert_has_id() + + # Do not pass the force parameter to the upper level because + # reloading files without MediaInfo will fail. + return super().get() def getID(self, numeric: bool = False): """ @@ -526,10 +549,6 @@ 'The provided Claim instance is already used in an entity') self._assert_has_id() - if not hasattr(self, '_revid'): - # workaround for uninitialized mediainfo's - self._revid = self.file.latest_revision_id - self.repo.addClaim(self, claim, bot=bot, **kwargs) claim.on_item = self diff --git a/tests/file_tests.py b/tests/file_tests.py index 40cc008..744d45c 100755 --- a/tests/file_tests.py +++ b/tests/file_tests.py @@ -430,11 +430,7 @@ item = page.data_item() self.assertIsInstance(item, pywikibot.MediaInfo) - # Get fails as there is no mediainfo. - with self.assertRaises(NoWikibaseEntityError): - item.get() - - self.assertFalse(item.exists()) + self.assertTrue(item.exists()) self.assertEqual(f'M{page.pageid}', item.id) self.assertIsInstance( item.labels, pywikibot.page._collections.LanguageDict) -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1006184?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I9178e3fe5a3a1ccba439864891bb1834fb28b050 Gerrit-Change-Number: 1006184 Gerrit-PatchSet: 3 Gerrit-Owner: Zache-tool <kimmo.virtanen(a)gmail.com> Gerrit-Reviewer: Ipr1 <ilkka.prusi(a)gmail.com> Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [IMPR] Adopt error message from Python 3.13 for batched function
by jenkins-bot (Code Review) 01 Mar '24

01 Mar '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007894?usp=email ) Change subject: [IMPR] Adopt error message from Python 3.13 for batched function ...................................................................... [IMPR] Adopt error message from Python 3.13 for batched function Also add some doc tests Change-Id: I97068d5dd664d125b53b870464ecb1fdc8bb4a65 --- M pywikibot/backports.py 1 file changed, 21 insertions(+), 2 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/backports.py b/pywikibot/backports.py index e49e815..5286439 100644 --- a/pywikibot/backports.py +++ b/pywikibot/backports.py @@ -156,6 +156,14 @@ Traceback (most recent call last): ... StopIteration + >>> list(batched('ABCD', 2)) + [('A', 'B'), ('C', 'D')] + >>> list(batched('ABCD', 3, strict=False)) + [('A', 'B', 'C'), ('D',)] + >>> list(batched('ABCD', 3, strict=True)) + Traceback (most recent call last): + ... + ValueError: batched(): incomplete batch .. seealso:: :python:`itertools.batched <library/itertools.html#itertools.batched>`, @@ -167,9 +175,9 @@ :param n: How many items of the iterable to get in one chunk :param strict: raise a ValueError if the final batch is shorter than *n*. - :raise ValueError: the final batch is shorter than *n*. + :raise ValueError: batched(): incomplete batch """ - msg = f'The final batch is shorter than n={n}' + msg = 'batched(): incomplete batch' if PYTHON_VERSION < (3, 12): group = [] for item in iterable: -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007894?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I97068d5dd664d125b53b870464ecb1fdc8bb4a65 Gerrit-Change-Number: 1007894 Gerrit-PatchSet: 3 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...i18n[master]: Localisation updates from https://translatewiki.net.
by jenkins-bot (Code Review) 29 Feb '24

29 Feb '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/i18n/+/1007594?usp=email ) Change subject: Localisation updates from https://translatewiki.net. ...................................................................... Localisation updates from https://translatewiki.net. Change-Id: I8f0fcaed56f69277dfc681a8f81dae4c93b2dd7b --- M checkimages/ko.json M replicate_wiki/ko.json 2 files changed, 15 insertions(+), 0 deletions(-) Approvals: jenkins-bot: Verified L10n-bot: Looks good to me, approved diff --git a/checkimages/ko.json b/checkimages/ko.json index 6ca5714..b2928df 100644 --- a/checkimages/ko.json +++ b/checkimages/ko.json @@ -1,6 +1,7 @@ { "@metadata": { "authors": [ + "Dr1t jg", "Nicolas Dumazet", "Revi", "Ykhwong", @@ -12,6 +13,9 @@ "checkimages-doubles-file-comment": "봇: 파일이 이미 공용에 존재하므로 삭제될 수 있음", "checkimages-doubles-head": "중복 파일", "checkimages-doubles-talk-comment": "봇: 파일이 이미 공용에 존재함을 알림", + "checkimages-doubles-talk-text": "%(upload)s를 올려줘서 고마워. 그런데 이 파일은 %(image)s의 복제본이야. 봇이 덜 사용되거나 가장 최근의 파일을 복제본로 표시했어. 이 파일이 빠르게 제거되어야 함으로 표시하는 게 더 적절하다고 여기면, 복제본 파일을 제거하고 남겨질 파일에 제거 틀을 없애도 돼. 이것은 %(bot)s의 자동화된 문자임.", + "checkimages-forced-mode": "('''강제 모드''')", + "checkimages-has-duplicates": "다음의 복제본이 있음%(force)s:", "checkimages-log-comment": "봇: 기록 업데이트", "checkimages-no-license-head": "라이선스 없는 이미지", "checkimages-source-tag-comment": "봇: 새로 업로드된 태그 없는 파일을 표시", diff --git a/replicate_wiki/ko.json b/replicate_wiki/ko.json index 0f55c72..480155c 100644 --- a/replicate_wiki/ko.json +++ b/replicate_wiki/ko.json @@ -1,10 +1,12 @@ { "@metadata": { "authors": [ + "Dr1t jg", "Ykhwong", "그냥기여자" ] }, "replicate_wiki-headline": "원본과 다른 페이지", + "replicate_wiki-missing-users": "여기에 없는 원본 관리자", "replicate_wiki-same-pages": "중요한 모든 문서가 동일합니다" } -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/i18n/+/1007594?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/i18n Gerrit-Branch: master Gerrit-Change-Id: I8f0fcaed56f69277dfc681a8f81dae4c93b2dd7b Gerrit-Change-Number: 1007594 Gerrit-PatchSet: 1 Gerrit-Owner: L10n-bot <l10n-bot(a)translatewiki.net> Gerrit-Reviewer: L10n-bot <l10n-bot(a)translatewiki.net> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [IMPR] backport itertools.batched from Python 3.13
by jenkins-bot (Code Review) 29 Feb '24

29 Feb '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007565?usp=email ) Change subject: [IMPR] backport itertools.batched from Python 3.13 ...................................................................... [IMPR] backport itertools.batched from Python 3.13 Python 3.13 batched has a strict option like our own itertools.itergroup function. Change-Id: Id3a334ac60ad20128ff14224fc1fc0eb2f5ac52b --- M pywikibot/backports.py M pywikibot/tools/itertools.py 2 files changed, 41 insertions(+), 14 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/backports.py b/pywikibot/backports.py index b7a2dab..e49e815 100644 --- a/pywikibot/backports.py +++ b/pywikibot/backports.py @@ -5,7 +5,7 @@ instead. The *SimpleQueue* queue; use ``queue.SimpleQueue`` instead. """ # -# (C) Pywikibot team, 2014-2023 +# (C) Pywikibot team, 2014-2024 # # Distributed under the terms of the MIT license. # @@ -135,11 +135,13 @@ # gh-98363 -if PYTHON_VERSION < (3, 12) or SPHINX_RUNNING: - def batched(iterable, n: int) -> Generator[tuple, None, None]: +if PYTHON_VERSION < (3, 13) or SPHINX_RUNNING: + def batched(iterable, n: int, *, + strict: bool = False) -> Generator[tuple, None, None]: """Batch data from the *iterable* into tuples of length *n*. - .. note:: The last batch may be shorter than *n*. + .. note:: The last batch may be shorter than *n* if *strict* is + True or raise a ValueError otherwise. Example: @@ -159,17 +161,32 @@ <library/itertools.html#itertools.batched>`, backported from Python 3.12. .. versionadded:: 8.2 + .. versionchanged:: 9.0 + Added *strict* option, backported from Python 3.13 :param n: How many items of the iterable to get in one chunk + :param strict: raise a ValueError if the final batch is shorter + than *n*. + :raise ValueError: the final batch is shorter than *n*. """ - group = [] - for item in iterable: - group.append(item) - if len(group) == n: + msg = f'The final batch is shorter than n={n}' + if PYTHON_VERSION < (3, 12): + group = [] + for item in iterable: + group.append(item) + if len(group) == n: + yield tuple(group) + group.clear() + if group: + if strict: + raise ValueError(msg) yield tuple(group) - group.clear() - if group: - yield tuple(group) + else: # PYTHON_VERSION == (3, 12) + from itertools import batched as _batched + for group in _batched(iterable, n): + if strict and len(group) < n: + raise ValueError(msg) + yield group else: from itertools import batched # type: ignore[no-redef] diff --git a/pywikibot/tools/itertools.py b/pywikibot/tools/itertools.py index 4a17c43..41fc5ca 100644 --- a/pywikibot/tools/itertools.py +++ b/pywikibot/tools/itertools.py @@ -60,9 +60,7 @@ not divisible by `size`. :raises ValueError: iterable is not divisible by size """ - for group in batched(iterable, size): - if strict and len(group) < size: - raise ValueError('iterable is not divisible by size.') + for group in batched(iterable, size, strict=strict): yield list(group) -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1007565?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Id3a334ac60ad20128ff14224fc1fc0eb2f5ac52b Gerrit-Change-Number: 1007565 Gerrit-PatchSet: 4 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

[Gerrit] ...core[master]: [IMPR] Raise APIError if the same error comes twice within submit loop
by jenkins-bot (Code Review) 27 Feb '24

27 Feb '24

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1004618?usp=email ) Change subject: [IMPR] Raise APIError if the same error comes twice within submit loop ...................................................................... [IMPR] Raise APIError if the same error comes twice within submit loop api.Request.submit handles API errors within a loop but in some circumstances the loop will never leaved if the same error comes again and again. Therefore raise APIError with the previous API error if the same error occured twice in the same submit loop- Bug: T357870 Change-Id: Id9f140c9d8815ef622d47cd90a29518f23665a4a --- M pywikibot/data/api/_requests.py 1 file changed, 42 insertions(+), 1 deletion(-) Approvals: JJMC89: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api/_requests.py b/pywikibot/data/api/_requests.py index bc40eec..8facad1 100644 --- a/pywikibot/data/api/_requests.py +++ b/pywikibot/data/api/_requests.py @@ -1,6 +1,6 @@ """Objects representing API requests.""" # -# (C) Pywikibot team, 2007-2023 +# (C) Pywikibot team, 2007-2024 # # Distributed under the terms of the MIT license. # @@ -945,18 +945,36 @@ self._params['token'] = tokens return True + def wait(self, delay: int | None = None) -> None: + """Determine how long to wait after a failed request. + + Also reset last API error with wait cycles. + + .. versionadded: 9.0 + + :param delay: Minimum time in seconds to wait. Overwrites + ``retry_wait`` variable if given. The delay doubles each + retry until ``retry_max`` seconds is reached. + """ + self.last_error = dict.fromkeys(['code', 'info']) + super().wait(delay) + def submit(self) -> dict: """Submit a query and parse the response. .. versionchanged:: 8.0.4 in addition to *readapidenied* also try to login when API response is *notloggedin*. + .. versionchanged:: 9.0 + Raise :exc:`pywikibot.exceptions.APIError` if the same error + comes twice in a row within the loop. :return: a dict containing data retrieved from api.php """ self._add_defaults() use_get = self._use_get() retries = 0 + self.last_error = dict.fromkeys(['code', 'info']) while True: paramstring = self._http_param_string() @@ -1003,6 +1021,11 @@ code = error.setdefault('code', 'Unknown') info = error.setdefault('info', None) + if (code == self.last_error['code'] + and info == self.last_error['info']): + raise pywikibot.exceptions.APIError(**self.last_error) + self.last_error = error + if not self._logged_in(code): continue @@ -1019,6 +1042,8 @@ lag = float(lag['lag']) if lag else 0.0 self.site.throttle.lag(lag * retries) + # reset last error + self.last_error = dict.fromkeys(['code', 'info']) continue if code == 'help' and self.action == 'help': @@ -1060,6 +1085,7 @@ pywikibot.error(f'Retrying failed {msg}') continue raise NoUsernameError(f'Failed {msg}') + if code == 'cirrussearch-too-busy-error': # T170647 self.wait() continue -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1004618?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Id9f140c9d8815ef622d47cd90a29518f23665a4a Gerrit-Change-Number: 1004618 Gerrit-PatchSet: 4 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: JJMC89 <JJMC89.Wikimedia(a)gmail.com> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits