jenkins-bot has submitted this change and it was merged.

Change subject: Add Wikidata support to isbn.py script ......................................................................

Add Wikidata support to isbn.py script

The support is done via a separate Bot class. It can find ISBN-10 and ISBN-13 property IDs or they can be provided manually by the user.

Also, the patch adds support for -always option in WikidataBot when using new method WikidataBot.userEditEntity.

Bug: T85242 Change-Id: I38cf459d78eb02102da6a169c9c0633ee95b1f3b --- M pywikibot/bot.py M scripts/isbn.py M tests/isbn_tests.py 3 files changed, 259 insertions(+), 12 deletions(-)

Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/bot.py b/pywikibot/bot.py index d56b526..0bb79ba 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -1006,19 +1006,36 @@ if 'comment' in kwargs: pywikibot.output(u'Comment: %s' % kwargs['comment'])

+ page.text = newtext + self._save_page(page, page.save, **kwargs) + + def _save_page(self, page, func, *args, **kwargs): + """ + Helper function to handle page save-related option error handling. + + @param page: currently edited page + @param func: the function to call + @param args: passed to the function + @param kwargs: passed to the function + @kwarg ignore_server_errors: if True, server errors will be reported + and ignored (default: False) + @kwtype ignore_server_errors: bool + @kwarg ignore_save_related_errors: if True, errors related to + page save will be reported and ignored (default: False) + @kwtype ignore_save_related_errors: bool + """ if not self.user_confirm('Do you want to accept these changes?'): return

if 'async' not in kwargs and self.getOption('always'): kwargs['async'] = True

- page.text = newtext - - ignore_save_related_errors = kwargs.pop('ignore_save_related_errors', False) + ignore_save_related_errors = kwargs.pop('ignore_save_related_errors', + False) ignore_server_errors = kwargs.pop('ignore_server_errors', False)

try: - page.save(**kwargs) + func(*args, **kwargs) except pywikibot.PageSaveRelatedError as e: if not ignore_save_related_errors: raise @@ -1164,6 +1181,67 @@ self.source_values[family_code][source_lang] = pywikibot.ItemPage(self.repo, family[source_lang])

+ def get_property_by_name(self, property_name): + """ + Find given property and return its ID. + + Method first uses site.search() and if the property isn't found, then + asks user to provide the property ID. + + @param property_name: property to find + @type property_name: str + """ + ns = self.site.data_repository().property_namespace + for page in self.site.search(property_name, step=1, total=1, + namespaces=ns): + page = pywikibot.PropertyPage(self.site.data_repository(), + page.title()) + pywikibot.output(u"Assuming that %s property is %s." % + (property_name, page.id)) + return page.id + return pywikibot.input(u'Property %s was not found. Please enter the ' + u'property ID (e.g. P123) of it:' + % property_name).upper() + + def user_edit_entity(self, item, data=None, **kwargs): + """ + Edit entity with data provided, with user confirmation as required. + + @param item: page to be edited + @type item: ItemPage + @param data: data to be saved, or None if the diff should be created + automatically + @kwarg summary: revision comment, passed to ItemPage.editEntity + @kwtype summary: str + @kwarg show_diff: show changes between oldtext and newtext (default: + True) + @kwtype show_diff: bool + @kwarg ignore_server_errors: if True, server errors will be reported + and ignored (default: False) + @kwtype ignore_server_errors: bool + @kwarg ignore_save_related_errors: if True, errors related to + page save will be reported and ignored (default: False) + @kwtype ignore_save_related_errors: bool + """ + self.current_page = item + + show_diff = kwargs.pop('show_diff', True) + if show_diff: + if data is None: + diff = item.toJSON(diffto=( + item._content if hasattr(item, '_content') else None)) + else: + diff = pywikibot.WikibasePage._normalizeData(data) + pywikibot.output(json.dumps(diff, indent=4, sort_keys=True)) + + if 'summary' in kwargs: + pywikibot.output(u'Change summary: %s' % kwargs['summary']) + + # TODO async in editEntity should actually have some effect (bug T86074) + # TODO PageSaveRelatedErrors should be actually raised in editEntity + # (bug T86083) + self._save_page(item, item.editEntity, data, **kwargs) + def getSource(self, site): """ Create a Claim usable as a source for Wikibase statements. diff --git a/scripts/isbn.py b/scripts/isbn.py index 54b2eb5..412214e 100755 --- a/scripts/isbn.py +++ b/scripts/isbn.py @@ -27,6 +27,13 @@

-always Don't prompt you for each replacement.

+-prop-isbn-10 Sets ISBN-10 property ID, so it's not tried to be found + automatically. + The usage is as follows: -prop-isbn-10:propid + +-prop-isbn-13 Sets ISBN-13 property ID. The format and purpose is the + same as in -prop-isbn-10. + """ # # (C) Pywikibot team, 2009-2014 @@ -38,7 +45,7 @@

import re import pywikibot -from pywikibot import i18n, pagegenerators, Bot +from pywikibot import i18n, pagegenerators, Bot, WikidataBot

docuReplacements = { '&params;': pagegenerators.parameterHelp, @@ -1415,6 +1422,89 @@ self.treat(page)

+class IsbnWikibaseBot(WikidataBot): + + """ISBN bot to be run on Wikibase sites.""" + + def __init__(self, generator, **kwargs): + self.availableOptions.update({ + 'to13': False, + 'format': False, + }) + self.isbn_10_prop_id = kwargs.pop('prop-isbn-10', None) + self.isbn_13_prop_id = kwargs.pop('prop-isbn-13', None) + + super(IsbnWikibaseBot, self).__init__(use_from_page=None, **kwargs) + + self.generator = generator + if self.isbn_10_prop_id is None: + self.isbn_10_prop_id = self.get_property_by_name('ISBN-10') + if self.isbn_13_prop_id is None: + self.isbn_13_prop_id = self.get_property_by_name('ISBN-13') + self.comment = i18n.twtranslate(pywikibot.Site(), 'isbn-formatting') + + def treat(self, page, item): + change_messages = [] + + if self.isbn_10_prop_id in item.claims: + for claim in item.claims[self.isbn_10_prop_id]: + try: + isbn = getIsbn(claim.getTarget()) + except InvalidIsbnException as e: + pywikibot.output(e) + continue + + old_code = claim.getTarget() + + if self.getOption('format'): + isbn.format() + + if self.getOption('to13'): + isbn = isbn.toISBN13() + + item.claims[claim.getID()].remove(claim) + claim = pywikibot.Claim(self.repo, self.isbn_13_prop_id) + claim.setTarget(isbn.code) + if self.isbn_13_prop_id in item.claims: + item.claims[self.isbn_13_prop_id].append(claim) + else: + item.claims[self.isbn_13_prop_id] = [claim] + change_messages.append('Changing %s (%s) to %s (%s)' % + (self.isbn_10_prop_id, old_code, + self.isbn_13_prop_id, isbn.code)) + continue + + if old_code == isbn.code: + continue + claim.setTarget(isbn.code) + change_messages.append('Changing %s (%s --> %s)' % + (self.isbn_10_prop_id, old_code, + isbn.code)) + + # -format is the only option that has any effect on ISBN13 + if self.getOption('format') and self.isbn_13_prop_id in item.claims: + for claim in item.claims[self.isbn_13_prop_id]: + try: + isbn = getIsbn(claim.getTarget()) + except InvalidIsbnException as e: + pywikibot.output(e) + continue + + old_code = claim.getTarget() + isbn.format() + if old_code == isbn.code: + continue + change_messages.append( + 'Changing %s (%s --> %s)' % (self.isbn_13_prop_id, + claim.getTarget(), isbn.code)) + claim.setTarget(isbn.code) + + if change_messages: + self.current_page = item + pywikibot.output('\n'.join(change_messages)) + self.user_edit_entity(item, summary=self.comment) + + def main(*args): """ Process command line arguments and invoke bot. @@ -1430,8 +1520,20 @@ local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory()

+ # Check whether we're running on Wikibase site or not + # FIXME: See T85483 and run() in WikidataBot + site = pywikibot.Site() + data_site = site.data_repository() + use_wikibase = (data_site is not None and + data_site.family == site.family and + data_site.code == site.code) + for arg in local_args: - if arg.startswith('-') and arg[1:] in ('always', 'to13', 'format'): + if arg.startswith('-prop-isbn-10:'): + options[arg[1:len('-prop-isbn-10')]] = arg[len('-prop-isbn-10:'):] + elif arg.startswith('-prop-isbn-13:'): + options[arg[1:len('-prop-isbn-13')]] = arg[len('-prop-isbn-13:'):] + elif arg.startswith('-') and arg[1:] in ('always', 'to13', 'format'): options[arg[1:]] = True else: genFactory.handleArg(arg) @@ -1439,7 +1541,10 @@ gen = genFactory.getCombinedGenerator() if gen: preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = IsbnBot(preloadingGen, **options) + if use_wikibase: + bot = IsbnWikibaseBot(preloadingGen, **options) + else: + bot = IsbnBot(preloadingGen, **options) bot.run() else: pywikibot.showHelp() diff --git a/tests/isbn_tests.py b/tests/isbn_tests.py index cd2c513..26ec753 100644 --- a/tests/isbn_tests.py +++ b/tests/isbn_tests.py @@ -9,10 +9,15 @@

__version__ = '$Id$'

-from scripts.isbn import ISBN10, ISBN13, InvalidIsbnException as IsbnExc, \ - getIsbn, hyphenateIsbnNumbers, convertIsbn10toIsbn13, main -from tests.aspects import TestCase, unittest -from pywikibot import Bot +from scripts.isbn import ( + ISBN10, ISBN13, InvalidIsbnException as IsbnExc, + getIsbn, hyphenateIsbnNumbers, convertIsbn10toIsbn13, + main +) +from tests.aspects import ( + unittest, TestCase, WikibaseTestCase, ScriptMainTestCase +) +from pywikibot import Bot, Claim, ItemPage

class TestIsbn(TestCase): @@ -89,7 +94,7 @@ isbn.format)

-class TestIsbnBot(TestCase): +class TestIsbnBot(ScriptMainTestCase):

"""Test isbnbot with non-write patching (if the testpage exists)."""

@@ -123,5 +128,64 @@ TestIsbnBot.newtext = newtext

+class TestIsbnWikibaseBot(ScriptMainTestCase, WikibaseTestCase): + + """Test isbnbot on Wikibase site with non-write patching.""" + + family = 'wikidata' + code = 'test' + + @classmethod + def setUpClass(cls): + super(TestIsbnWikibaseBot, cls).setUpClass() + + # Check if the unit test item page and the property both exist + item_ns = cls.get_repo().item_namespace + for page in cls.get_site().search('IsbnWikibaseBotUnitTest', step=1, + total=1, namespaces=item_ns): + cls.test_page_qid = page.title() + item_page = ItemPage(cls.get_repo(), page.title()) + for pid, claims in item_page.get()['claims'].items(): + for claim in claims: + prop_page = pywikibot.PropertyPage(cls.get_repo(), + claim.getID()) + prop_page.get() + if ('ISBN-10' in prop_page.labels.values() and + claim.getTarget() == '097522980x'): + return + raise unittest.SkipTest( + u'%s: "ISBN-10" property was not found in ' + u'"IsbnWikibaseBotUnitTest" item page' % cls.__name__) + raise unittest.SkipTest( + u'%s: "IsbnWikibaseBotUnitTest" item page was not found' + % cls.__name__) + + def setUp(self): + TestIsbnWikibaseBot._original_setTarget = Claim.setTarget + Claim.setTarget = setTarget_dummy + TestIsbnWikibaseBot._original_editEntity = ItemPage.editEntity + ItemPage.editEntity = editEntity_dummy + super(TestIsbnWikibaseBot, self).setUp() + + def tearDown(self): + Claim.setTarget = TestIsbnWikibaseBot._original_setTarget + ItemPage.editEntity = TestIsbnWikibaseBot._original_editEntity + super(TestIsbnWikibaseBot, self).tearDown() + + def test_isbn(self): + main('-page:' + self.test_page_qid, '-always', '-format') + self.assertEqual(self.setTarget_value, '0-9752298-0-X') + main('-page:' + self.test_page_qid, '-always', '-to13') + self.assertTrue(self.setTarget_value, '978-0975229804') + + +def setTarget_dummy(self, value): + TestIsbnWikibaseBot.setTarget_value = value + TestIsbnWikibaseBot._original_setTarget(self, value) + + +def editEntity_dummy(self, data=None, **kwargs): + pass + if __name__ == "__main__": unittest.main()