Pywikibot-commits October 2014

pywikibot-commits@lists.wikimedia.org

1 participants
155 discussions

[Gerrit] Fix newpages default namespace - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Fix newpages default namespace ...................................................................... Fix newpages default namespace The ability to set namespaces for newitem.py, yet default to namespace 0 only was added for bug 67249, however this was inadvertantly removed in 60aceea. Bug: 72176 Change-Id: I729e84e83c6a4d5e4e69e877cbea9780176d1f27 --- M pywikibot/pagegenerators.py M tests/aspects.py M tests/pagegenerators_tests.py 3 files changed, 104 insertions(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index b79dfda..128f6c4 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -516,10 +516,11 @@ # partial workaround for bug 67249 # to use -namespace/ns with -newpages, -ns must be given before -newpages # otherwise default namespace is 0 + namespaces = self.namespaces or 0 total = 60 if len(arg) >= 10: total = int(arg[10:]) - gen = NewpagesPageGenerator(namespaces=self.namespaces, + gen = NewpagesPageGenerator(namespaces=namespaces, total=total, site=self.site) elif arg.startswith('-imagesused'): diff --git a/tests/aspects.py b/tests/aspects.py index 40edb6d..e1b2900 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -67,6 +67,62 @@ """ return self.assertRegexpMatches(*args, **kwargs) + def assertPageInNamespaces(self, page, namespaces): + """ + Assert that Pages is in namespaces. + + @param page: Page + @type page: Page + @param namespaces: expected namespaces + @type namespaces: int or set of int + """ + if isinstance(namespaces, int): + namespaces = set([namespaces]) + + self.assertIn(page.namespace(), namespaces, + "%s not in namespace %r" % (page, namespaces)) + + def assertPagesInNamespaces(self, gen, namespaces): + """ + Assert that generator returns Pages all in namespaces. + + @param gen: generator to iterate + @type gen: generator + @param namespaces: expected namespaces + @type namespaces: int or set of int + """ + if isinstance(namespaces, int): + namespaces = set([namespaces]) + + for page in gen: + self.assertPageInNamespaces(page, namespaces) + + def assertPagesInNamespacesAll(self, gen, namespaces, skip=False): + """ + Try to confirm that generator returns Pages for all namespaces. + + @param gen: generator to iterate + @type gen: generator + @param namespaces: expected namespaces + @type namespaces: int or set of int + @param count: maximum results to process + @type count: int + @param skip: skip test if not all namespaces found + @param skip: bool + """ + if isinstance(namespaces, int): + namespaces = set([namespaces]) + else: + assert(isinstance(namespaces, set)) + + page_namespaces = [page.namespace() for page in gen] + + if skip and set(page_namespaces) != namespaces: + raise unittest.SkipTest('Pages in namespaces %r not found.' + % list(namespaces - set(page_namespaces))) + else: + self.assertEqual(set(page_namespaces), namespaces) + class TestLoggingMixin(TestCaseBase): diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 96e5dfa..da4c390 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -196,6 +196,52 @@ self.assertTrue(all(isinstance(item, pywikibot.ItemPage) for item in gen)) +class TestFactoryGenerator(DefaultSiteTestCase): + + """Test pagegenerators.GeneratorFactory.""" + + def test_newpages_default(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-newpages') + gen = gf.getCombinedGenerator() + pages = set(gen) + self.assertEqual(len(pages), 60) + + def test_newpages_ns_default(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-newpages:10') + gen = gf.getCombinedGenerator() + self.assertPagesInNamespaces(gen, 0) + + def test_newpages_ns(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-ns:1') + gf.handleArg('-newpages:10') + gen = gf.getCombinedGenerator() + self.assertPagesInNamespaces(gen, 1) + + def test_recentchanges_ns_default(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-recentchanges:50') + gen = gf.getCombinedGenerator() + self.assertPagesInNamespacesAll(gen, set([0, 1, 2]), skip=True) + + def test_recentchanges_ns(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-ns:1') + gf.handleArg('-recentchanges:10') + gen = gf.getCombinedGenerator() + self.assertPagesInNamespaces(gen, 1) + + def test_recentchanges_ns_multi(self): + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-ns:1') + gf.handleArg('-ns:3') + gf.handleArg('-recentchanges:10') + gen = gf.getCombinedGenerator() + self.assertPagesInNamespaces(gen, set([1, 3])) + + if __name__ == "__main__": try: unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/167796 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I729e84e83c6a4d5e4e69e877cbea9780176d1f27 Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] port PageTitleFilterPageGenerator from compat - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: port PageTitleFilterPageGenerator from compat ...................................................................... port PageTitleFilterPageGenerator from compat changed re.match to re.search for more precise comparison bug: 55007 Change-Id: I3ff0bebff3e4bfc62fde290811d8d6504532e9c0 --- M pywikibot/pagegenerators.py 1 file changed, 26 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index b79dfda..fab7ccd 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -918,6 +918,32 @@ yield page +@deprecated_args(ignoreList='ignore_list') +def PageTitleFilterPageGenerator(generator, ignore_list): + """ + Yield only those pages are not listed in the ignore list. + + @param ignore_list: family names are mapped to dictionaries + in which language codes are mapped to lists of page titles + @type ignore_list: dict + + """ + def is_ignored(page): + if page.site.family.name in ignore_list and \ + page.site.code in ignore_list[page.site.family.name]: + for ig in ignore_list[page.site.family.name][page.site.code]: + if re.search(ig, page.title()): + return True + return False + + for page in generator: + if is_ignored(page): + if config.verbose_output: + pywikibot.output('Ignoring page %s' % page.title()) + else: + yield page + + def RedirectFilterPageGenerator(generator, no_redirects=True): """Yield pages from another generator that are redirects or not.""" for page in generator: -- To view, visit https://gerrit.wikimedia.org/r/135407 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I3ff0bebff3e4bfc62fde290811d8d6504532e9c0 Gerrit-PatchSet: 8 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] CosmeticChangesBot shows diff twice - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: CosmeticChangesBot shows diff twice ...................................................................... CosmeticChangesBot shows diff twice Rename CosmeticChangesToolkit option 'debug' to 'diff', as that is its only function. CosmeticChangesBot should call CosmeticChangesToolkit with the diff option disabled, as userPut will show the diff. Change-Id: Ia44ca916e5aaaef08f9b4ea5ebcdcd24cddf3f08 --- M scripts/cosmetic_changes.py 1 file changed, 6 insertions(+), 4 deletions(-) Approvals: John Vandenberg: Looks good to me, approved XZise: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/scripts/cosmetic_changes.py b/scripts/cosmetic_changes.py index f0c892d..a59b217 100755 --- a/scripts/cosmetic_changes.py +++ b/scripts/cosmetic_changes.py @@ -79,6 +79,7 @@ import isbn from pywikibot import config, i18n, textlib, pagegenerators, Bot from pywikibot.page import url2unicode +from pywikibot.tools import deprecate_arg warning = """ ATTENTION: You can run this script as a stand-alone for testing purposes. @@ -159,10 +160,11 @@ """Cosmetic changes toolkit.""" - def __init__(self, site, debug=False, redirect=False, namespace=None, + @deprecate_arg('debug', 'diff') + def __init__(self, site, diff=False, redirect=False, namespace=None, pageTitle=None, ignore=CANCEL_ALL): self.site = site - self.debug = debug + self.diff = diff self.redirect = redirect self.namespace = namespace self.template = (self.namespace == 10) @@ -236,7 +238,7 @@ else: raise else: - if self.debug: + if self.diff: pywikibot.showDiff(text, new_text) return new_text @@ -907,7 +909,7 @@ def treat(self, page): try: self.current_page = page - ccToolkit = CosmeticChangesToolkit(page.site, debug=True, + ccToolkit = CosmeticChangesToolkit(page.site, diff=False, namespace=page.namespace(), pageTitle=page.title(), ignore=self.getOption('ignore')) -- To view, visit https://gerrit.wikimedia.org/r/169652 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia44ca916e5aaaef08f9b4ea5ebcdcd24cddf3f08 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] Cosmetic changes fixup - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Cosmetic changes fixup ...................................................................... Cosmetic changes fixup f0543d6 heavily revised to flow and error control of the script cosmetic_changes. However a bug meant the changes made by the change methods were discarded. Change-Id: I2adecb09876bdf64ff4e9e03dd095368fa6135b2 --- M scripts/cosmetic_changes.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: XZise: Looks good to me, approved Dalba: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/scripts/cosmetic_changes.py b/scripts/cosmetic_changes.py index f0c892d..07f9c33 100755 --- a/scripts/cosmetic_changes.py +++ b/scripts/cosmetic_changes.py @@ -206,7 +206,7 @@ pywikibot.exception(e) else: raise - return text if result is None else text + return text if result is None else result @staticmethod def isbn_execute(text): -- To view, visit https://gerrit.wikimedia.org/r/169651 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2adecb09876bdf64ff4e9e03dd095368fa6135b2 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] [FIX] DequeGenerator tests: always add talk page - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] DequeGenerator tests: always add talk page ...................................................................... [FIX] DequeGenerator tests: always add talk page The tests for wikidata and wiktionary fail because their main pages are not in the main namespace. Change-Id: I46c4e507f0839023fe355161bb299e74c42c5367 --- M tests/pagegenerators_tests.py 1 file changed, 3 insertions(+), 3 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index dd85f1b..96e5dfa 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -173,14 +173,14 @@ for page in gen: pages_out.append(page) # Add a page to the generator - if page.namespace() == 0: + if not page.isTalkPage(): pages.extend([page.toggleTalkPage()]) self.assertTrue(all(isinstance(page, pywikibot.Page) for page in pages_out)) - self.assertEqual(len(pages_out), 2) - self.assertEqual(pages_out[1].namespace(), 1) self.assertIn(mainpage, pages_out) self.assertIn(mainpage.toggleTalkPage(), pages_out) + self.assertEqual(len(pages_out), 2) + self.assertTrue(pages_out[1].isTalkPage()) class TestPreloadingItemGenerator(WikidataTestCase): -- To view, visit https://gerrit.wikimedia.org/r/169647 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I46c4e507f0839023fe355161bb299e74c42c5367 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] [FEAT] Diff: Use PatchManager for showDiff - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] Diff: Use PatchManager for showDiff ...................................................................... [FEAT] Diff: Use PatchManager for showDiff This uses PatchManager for pywikibot.showDiff and also includes now a header. It also fixes the missing space in front of a line if it was added or removed. This makes it consistent with lines where only parts of it were changed which use difflib.ndiff. And it fixes a crash if both texts are equal and not empty. Also a minor change that the state which stores if it ends with the default color is now represented as a boolean. Change-Id: Ic59df944b4c64b8536a5178e1aabc0b1adfeab47 --- M pywikibot/__init__.py M pywikibot/diff.py 2 files changed, 18 insertions(+), 65 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index d70dc86..fc1a708 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -9,7 +9,6 @@ __version__ = '$Id$' import datetime -import difflib import math import re import sys @@ -49,6 +48,7 @@ from pywikibot.tools import UnicodeMixin, redirect_func from pywikibot.i18n import translate from pywikibot.data.api import UploadWarning +from pywikibot.diff import PatchManager import pywikibot.textlib as textlib import pywikibot.tools @@ -602,60 +602,7 @@ The differences are highlighted (only on compatible systems) to show which changes were made. """ - # This is probably not portable to non-terminal interfaces.... - # For information on difflib, see http://pydoc.org/2.1/difflib.html - color = { - '+': 'lightgreen', - '-': 'lightred', - } - diff = u'' - colors = [] - # This will store the last line beginning with + or -. - lastline = None - # For testing purposes only: show original, uncolored diff - # for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): - # print line - for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): - if line.startswith('?'): - # initialize color vector with None, which means default color - lastcolors = [None for c in lastline] - # colorize the + or - sign - lastcolors[0] = color[lastline[0]] - # colorize changed parts in red or green - for i in range(min(len(line), len(lastline))): - if line[i] != ' ': - lastcolors[i] = color[lastline[0]] - diff += lastline + '\n' - # append one None (default color) for the newline character - colors += lastcolors + [None] - elif lastline: - diff += lastline + '\n' - # colorize the + or - sign only - lastcolors = [None for c in lastline] - lastcolors[0] = color[lastline[0]] - colors += lastcolors + [None] - lastline = None - if line[0] in ('+', '-'): - lastline = line - # there might be one + or - line left that wasn't followed by a ? line. - if lastline: - diff += lastline + '\n' - # colorize the + or - sign only - lastcolors = [None for c in lastline] - lastcolors[0] = color[lastline[0]] - colors += lastcolors + [None] - - result = u'' - lastcolor = None - for i in range(len(diff)): - if colors[i] != lastcolor: - if lastcolor is None: - result += '\03{%s}' % colors[i] - else: - result += '\03{default}' - lastcolor = colors[i] - result += diff[i] - output(result) + PatchManager(oldtext, newtext).print_hunks() # Throttle and thread handling diff --git a/pywikibot/diff.py b/pywikibot/diff.py index 7ac30dc..e118770 100644 --- a/pywikibot/diff.py +++ b/pywikibot/diff.py @@ -77,15 +77,17 @@ return l for tag, i1, i2, j1, j2 in self.group: + # equal/delete/insert add additional space after the sign as it's + # what difflib.ndiff does do too. if tag == 'equal': for line in self.a[i1:i2]: - yield ' ' + check_line(line) + yield ' ' + check_line(line) if tag in ('delete'): for line in self.a[i1:i2]: - yield '-' + check_line(line) + yield '- ' + check_line(line) if tag in ('insert'): for line in self.b[j1:j2]: - yield '+' + check_line(line) + yield '+ ' + check_line(line) if tag in ('replace'): for line in difflib.ndiff(self.a[i1:i2], self.b[j1:j2]): yield check_line(line) @@ -132,20 +134,20 @@ return line colored_line = u'' - state = 'Close' + color_closed = True for char, char_ref in zip_longest(line, line_ref.strip(), fillvalue=' '): char_tagged = char - if state == 'Close': + if color_closed: if char_ref != ' ': char_tagged = '\03{%s}%s' % (self.colors[color], char) - state = 'Open' - elif state == 'Open': + color_closed = False + else: if char_ref == ' ': char_tagged = '\03{default}%s' % char - state = 'Close' + color_closed = True colored_line += char_tagged - if state == 'Open': + if not color_closed: colored_line += '\03{default}' return colored_line @@ -228,11 +230,15 @@ # there is a section of unchanged text at the end of a, b. if i2 < len(self.a): - rng = (-1, (last[2], len(self.a)), (-1, -1)) + rng = (-1, (i2, len(self.a)), (-1, -1)) blocks.append(rng) return blocks + def print_hunks(self): + for hunk in self.hunks: + pywikibot.output(hunk.header + hunk.diff_text) + def review_hunks(self): "Review hunks." -- To view, visit https://gerrit.wikimedia.org/r/168967 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic59df944b4c64b8536a5178e1aabc0b1adfeab47 Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] [IMPROV] TestBaseCase: Only add backward compatible aliases ... - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [IMPROV] TestBaseCase: Only add backward compatible aliases if needed ...................................................................... [IMPROV] TestBaseCase: Only add backward compatible aliases if needed Instead of assuming that Python 2 won't support currently Python 3 specific methods this just adds a wrapper for the old name if the new name isn't implemented. Change-Id: I1022d27fcecdc516b3ebf42dc3380458163b2aa4 --- M tests/aspects.py 1 file changed, 2 insertions(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/aspects.py b/tests/aspects.py index d6da7b8..40edb6d 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -49,7 +49,7 @@ """Base class for all tests.""" - if sys.version_info[0] == 2: + if not hasattr(unittest.TestCase, 'assertRaisesRegex'): def assertRaisesRegex(self, *args, **kwargs): """ Wrapper of unittest.assertRaisesRegexp for Python 2 unittest. @@ -58,6 +58,7 @@ """ return self.assertRaisesRegexp(*args, **kwargs) + if not hasattr(unittest.TestCase, 'assertRegex'): def assertRegex(self, *args, **kwargs): """ Wrapper of unittest.assertRegexpMatches for Python 2 unittest. -- To view, visit https://gerrit.wikimedia.org/r/169645 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1022d27fcecdc516b3ebf42dc3380458163b2aa4 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] Simplify initialisation logic - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Simplify initialisation logic ...................................................................... Simplify initialisation logic Change-Id: Ia8df25bc2d06c7a53c18e869aae87bac901321ac --- M scripts/makecat.py 1 file changed, 12 insertions(+), 8 deletions(-) Approvals: Nullzero: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/makecat.py b/scripts/makecat.py index cb2acf4..0b28870 100644 --- a/scripts/makecat.py +++ b/scripts/makecat.py @@ -235,10 +235,13 @@ except IOError: # File does not exist excludefile = codecs.open(filename, 'w', encoding=mysite.encoding()) + + # Get parent categories in order to `removeparent` try: parentcats = workingcat.categories() except pywikibot.Error: parentcats = [] + # Do not include articles already in subcats; only checking direct subcats subcatlist = list(workingcat.subcategories()) if subcatlist: @@ -247,14 +250,12 @@ artlist = list(cat.articles()) for page in artlist: checked[page] = page - list = [x for x in workingcat.articles()] - if list: - for pl in list: - checked[pl] = pl - list = pagegenerators.PreloadingGenerator(list) - for pl in list: - include(pl) - else: + + # Fetch articles in category, and mark as already checked (seen) + # If category is empty, ask user if they want to look for pages + # in a diferent category. + articles = list(workingcat.articles(content=True)) + if not articles: pywikibot.output( u"Category %s does not exist or is empty. Which page to start with?" % workingcatname) @@ -263,6 +264,9 @@ answer = workingcatname pywikibot.output(u'' + answer) pl = pywikibot.Page(mysite, answer) + articles = [pl] + + for pl in articles: checked[pl] = pl include(pl) -- To view, visit https://gerrit.wikimedia.org/r/169640 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia8df25bc2d06c7a53c18e869aae87bac901321ac Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Nullzero <nullzero.free(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] [IMPROV] Check version via > 2 or == 2 - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [IMPROV] Check version via > 2 or == 2 ...................................................................... [IMPROV] Check version via > 2 or == 2 This unifies the Python version check. Instead of different kind of checks (> 2, == 3, >= 3 or == 2, < 3, <= 2) this always uses the first variant (> 2 and == 2) and if it's like a switch it always does the Python 3 specific code first and the Python 2 specific code in the else branch. Change-Id: I826fd50b7343ff5f6e775936957cdbb5e18cd40f --- M generate_family_file.py M pwb.py M pywikibot/__init__.py M pywikibot/comms/http.py M pywikibot/comms/threadedhttp.py M pywikibot/data/wikidataquery.py M pywikibot/family.py M pywikibot/i18n.py M pywikibot/page.py M pywikibot/textlib.py M pywikibot/tools.py M pywikibot/userinterfaces/terminal_interface_base.py M pywikibot/weblib.py M tests/aspects.py M tests/http_tests.py M tests/weblib_tests.py 16 files changed, 62 insertions(+), 68 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/generate_family_file.py b/generate_family_file.py index 78e663f..f98de19 100644 --- a/generate_family_file.py +++ b/generate_family_file.py @@ -21,19 +21,20 @@ import codecs from collections import defaultdict from distutils.version import LooseVersion as V - -if sys.version_info[0] == 3: - raw_input = input +import json # creating & retrieving urls -if sys.version_info[0] == 2: - from urlparse import urlparse, urljoin - import urllib2 - from urllib2 import HTTPError -else: +if sys.version_info[0] > 2: from urllib.parse import urlparse, urljoin from urllib.error import HTTPError import urllib.request as urllib2 + from html.parser import HTMLParser + raw_input = input +else: + from urlparse import urlparse, urljoin + import urllib2 + from urllib2 import HTTPError + from HTMLParser import HTMLParser def urlopen(url): @@ -43,20 +44,13 @@ ' - https://www.mediawiki.org/wiki/Pywikibot'}) uo = urllib2.urlopen(req) try: - if sys.version_info[0] == 2: - uo.charset = uo.headers.getfirstmatchingheader('Content-Type')[0].strip().split('charset=')[1] - else: + if sys.version_info[0] > 2: uo.charset = uo.headers.get_content_charset() + else: + uo.charset = uo.headers.getfirstmatchingheader('Content-Type')[0].strip().split('charset=')[1] except IndexError: uo.charset = 'latin-1' return uo - -# parsing response data -import json -if sys.version_info[0] == 2: - from HTMLParser import HTMLParser -else: - from html.parser import HTMLParser class WikiHTMLPageParser(HTMLParser): diff --git a/pwb.py b/pwb.py index 16efbca..3a53ef2 100644 --- a/pwb.py +++ b/pwb.py @@ -56,10 +56,10 @@ main_mod = imp.new_module('__main__') sys.modules['__main__'] = main_mod main_mod.__file__ = filename - if sys.version_info[0] == 2: - main_mod.__builtins__ = sys.modules['__builtin__'] - else: + if sys.version_info[0] > 2: main_mod.builtins = sys.modules['builtins'] + else: + main_mod.__builtins__ = sys.modules['__builtin__'] # Set sys.argv and the first path element properly. old_argv = sys.argv diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index d70dc86..3dbfb2d 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -16,11 +16,11 @@ import threading import json -if sys.version_info[0] == 2: - from Queue import Queue -else: +if sys.version_info[0] > 2: from queue import Queue long = int +else: + from Queue import Queue # Use pywikibot. prefix for all in-package imports; this is to prevent # confusion with similarly-named modules in version 1 framework, for users diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index 8822c6f..92eeb5d 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -42,7 +42,14 @@ httplib2.__file__) sys.exit(1) -if sys.version_info[0] == 2: +if sys.version_info[0] > 2: + from ssl import SSLError as SSLHandshakeError + SSL_CERT_VERIFY_FAILED_MSG = "SSL: CERTIFICATE_VERIFY_FAILED" + import queue as Queue + import urllib.parse as urlparse + from http import cookiejar as cookielib + from urllib.parse import quote +else: if 'SSLHandshakeError' in httplib2.__dict__: from httplib2 import SSLHandshakeError elif httplib2.__version__ == '0.6.0': @@ -57,13 +64,6 @@ import urlparse import cookielib from urllib2 import quote -else: - from ssl import SSLError as SSLHandshakeError - SSL_CERT_VERIFY_FAILED_MSG = "SSL: CERTIFICATE_VERIFY_FAILED" - import queue as Queue - import urllib.parse as urlparse - from http import cookiejar as cookielib - from urllib.parse import quote from pywikibot import config from pywikibot.exceptions import FatalServerError, Server504Error diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py index c63daa4..4279454 100644 --- a/pywikibot/comms/threadedhttp.py +++ b/pywikibot/comms/threadedhttp.py @@ -25,12 +25,12 @@ import re import threading -if sys.version_info[0] == 2: - import cookielib - from urllib import splittype, splithost, unquote -else: +if sys.version_info[0] > 2: from http import cookiejar as cookielib from urllib.parse import splittype, splithost, unquote +else: + import cookielib + from urllib import splittype, splithost, unquote import pywikibot from pywikibot import config diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py index f808053..4e4f583 100644 --- a/pywikibot/data/wikidataquery.py +++ b/pywikibot/data/wikidataquery.py @@ -7,19 +7,19 @@ import json import sys -if sys.version_info[0] == 2: - from urllib2 import quote -else: +if sys.version_info[0] > 2: from urllib.parse import quote -from pywikibot.comms import http +else: + from urllib2 import quote import pickle import os import hashlib import time import tempfile -from pywikibot.page import ItemPage, PropertyPage, Claim import pywikibot +from pywikibot.comms import http +from pywikibot.page import ItemPage, PropertyPage, Claim from pywikibot import config diff --git a/pywikibot/family.py b/pywikibot/family.py index 2beb963..645138a 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -14,10 +14,10 @@ import collections import imp -if sys.version_info[0] == 2: - from urlparse import urlparse -else: +if sys.version_info[0] > 2: from urllib.parse import urlparse +else: + from urlparse import urlparse import pywikibot from pywikibot import config2 as config diff --git a/pywikibot/i18n.py b/pywikibot/i18n.py index 180b9f4..7e5b2db 100644 --- a/pywikibot/i18n.py +++ b/pywikibot/i18n.py @@ -21,7 +21,7 @@ import pywikibot from . import config2 as config -if sys.version_info[0] == 3: +if sys.version_info[0] > 2: basestring = (str, ) PLURAL_PATTERN = '{{PLURAL:(?:%$)?([^$]*?)(?:\)d)?\|(.*?)}}' diff --git a/pywikibot/page.py b/pywikibot/page.py index ef678d0..7025a9a 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -29,16 +29,16 @@ except ImportError: from ordereddict import OrderedDict -if sys.version_info[0] == 2: - import htmlentitydefs - from urllib import quote as quote_from_bytes, unquote as unquote_to_bytes - from urllib import urlopen -else: +if sys.version_info[0] > 2: unicode = basestring = str long = int from html import entities as htmlentitydefs from urllib.parse import quote_from_bytes, unquote_to_bytes from urllib.request import urlopen +else: + import htmlentitydefs + from urllib import quote as quote_from_bytes, unquote as unquote_to_bytes + from urllib import urlopen import pywikibot from pywikibot import config diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py index 87824e3..5c02e4e 100644 --- a/pywikibot/textlib.py +++ b/pywikibot/textlib.py @@ -22,12 +22,12 @@ import re import sys -if sys.version_info[0] == 2: - from HTMLParser import HTMLParser -else: +if sys.version_info[0] > 2: from html.parser import HTMLParser basestring = (str,) unicode = str +else: + from HTMLParser import HTMLParser from pywikibot import config2 as config import pywikibot diff --git a/pywikibot/tools.py b/pywikibot/tools.py index 6064d86..f4e3479 100644 --- a/pywikibot/tools.py +++ b/pywikibot/tools.py @@ -41,7 +41,7 @@ """Mixin class to add __str__ method in Python 2 or 3.""" - if sys.version_info[0] >= 3: + if sys.version_info[0] > 2: def __str__(self): return self.__unicode__() else: diff --git a/pywikibot/userinterfaces/terminal_interface_base.py b/pywikibot/userinterfaces/terminal_interface_base.py index f86859f..ee234e2 100755 --- a/pywikibot/userinterfaces/terminal_interface_base.py +++ b/pywikibot/userinterfaces/terminal_interface_base.py @@ -103,7 +103,7 @@ line, count = colorTagR.subn('', line) if count > 0: line += ' ***' - if sys.version_info[0] < 3: + if sys.version_info[0] == 2: line = line.encode(self.encoding, 'replace') targetStream.write(line) @@ -173,7 +173,7 @@ self._print(text, targetStream) def _raw_input(self): - if sys.version_info[0] >= 3: + if sys.version_info[0] > 2: return input() else: return raw_input() @@ -201,7 +201,7 @@ text = self._raw_input() except KeyboardInterrupt: raise pywikibot.QuitKeyboardInterrupt() - if sys.version_info[0] < 3: + if sys.version_info[0] == 2: text = text.decode(self.encoding) return text diff --git a/pywikibot/weblib.py b/pywikibot/weblib.py index 15b53ef..b608e6d 100644 --- a/pywikibot/weblib.py +++ b/pywikibot/weblib.py @@ -8,10 +8,10 @@ __version__ = '$Id$' import sys -if sys.version_info[0] == 2: - from urllib import urlencode -else: +if sys.version_info[0] > 2: from urllib.parse import urlencode +else: + from urllib import urlencode from pywikibot.comms import http diff --git a/tests/aspects.py b/tests/aspects.py index 0f3b9f1..eb418d9 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -47,7 +47,7 @@ """Base class for all tests.""" - if sys.version_info[0] < 3: + if sys.version_info[0] == 2: def assertRaisesRegex(self, *args, **kwargs): """ Wrapper of unittest.assertRaisesRegexp for Python 2 unittest. diff --git a/tests/http_tests.py b/tests/http_tests.py index 19c8909..28bb87a 100644 --- a/tests/http_tests.py +++ b/tests/http_tests.py @@ -14,7 +14,7 @@ from tests.aspects import unittest, TestCase from tests.utils import expectedFailureIf -if sys.version_info[0] == 3: +if sys.version_info[0] > 2: unicode = str @@ -43,7 +43,7 @@ site=None, uri='https://www.omegawiki.org/') - @expectedFailureIf(sys.version_info[0] == 3) # bug 72236 + @expectedFailureIf(sys.version_info[0] > 2) # bug 72236 def test_https_ignore_cert_error(self): """Test http request function ignoring ssl bad certificate.""" # As the connection is cached, the above test will cause @@ -102,7 +102,7 @@ self.assertIsInstance(r[0]['status'], str) self.assertEqual(r[0]['status'], '200') - self.assertIsInstance(r[1], bytes if sys.version_info[0] >= 3 else str) + self.assertIsInstance(r[1], bytes if sys.version_info[0] > 2 else str) self.assertIn(b'<html lang="mul"', r[1]) self.assertEqual(int(r[0]['content-length']), len(r[1])) @@ -116,7 +116,7 @@ self.assertIsInstance(r[0]['status'], str) self.assertEqual(r[0]['status'], '200') - self.assertIsInstance(r[1], bytes if sys.version_info[0] >= 3 else str) + self.assertIsInstance(r[1], bytes if sys.version_info[0] > 2 else str) self.assertIn(b'<html lang="mul"', r[1]) self.assertEqual(int(r[0]['content-length']), len(r[1])) diff --git a/tests/weblib_tests.py b/tests/weblib_tests.py index 14e338a..f4916e7 100644 --- a/tests/weblib_tests.py +++ b/tests/weblib_tests.py @@ -9,10 +9,10 @@ import os import sys -if sys.version_info[0] == 2: - from urlparse import urlparse -else: +if sys.version_info[0] > 2: from urllib.parse import urlparse +else: + from urlparse import urlparse import pywikibot.weblib as weblib from tests.aspects import unittest, TestCase -- To view, visit https://gerrit.wikimedia.org/r/168969 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I826fd50b7343ff5f6e775936957cdbb5e18cd40f Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

[Gerrit] DequeGenerator - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: DequeGenerator ...................................................................... DequeGenerator Scripts may be written to dynamically populate a list of pages to work on. Ideally this list would be created using a generator, however that can require a large rewrite in the flow control. To simplify the porting of scripts, this changeset adds classes DequeGenerator and DequePreloadingGenerator classes that allow the page list to be extended during iteration. Implemented in makecat. Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088 --- M pywikibot/pagegenerators.py M pywikibot/tools.py M scripts/makecat.py M tests/pagegenerators_tests.py 4 files changed, 72 insertions(+), 22 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index eb8930a..b79dfda 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -28,7 +28,7 @@ import time import pywikibot from pywikibot import date, config, i18n -from pywikibot.tools import deprecated_args +from pywikibot.tools import deprecated_args, DequeGenerator from pywikibot.comms import http import pywikibot.data.wikidataquery as wdquery @@ -1143,6 +1143,18 @@ yield i +def DequePreloadingGenerator(generator, step=50): + assert(isinstance(generator, DequeGenerator)) + + while True: + page_count = min(len(generator), step) + if not page_count: + raise StopIteration + + for page in PreloadingGenerator(generator, page_count): + yield page + + def PreloadingItemGenerator(generator, step=50): """ Yield preloaded pages taken from another generator. diff --git a/pywikibot/tools.py b/pywikibot/tools.py index 6064d86..dcf4fd4 100644 --- a/pywikibot/tools.py +++ b/pywikibot/tools.py @@ -13,7 +13,7 @@ import time import inspect import re -from collections import Mapping +from collections import Mapping, deque from distutils.version import Version if sys.version_info[0] > 2: @@ -320,6 +320,26 @@ EMPTY_DEFAULT = EmptyDefault() + +class DequeGenerator(deque): + + """A generator that allows items to be added during generating.""" + + def __iter__(self): + """Return the object which will be iterated.""" + return self + + def next(self): + """Python 3 iterator method.""" + if len(self): + return self.popleft() + else: + raise StopIteration + + def __next__(self): + """Python 3 iterator method.""" + return self.next() + # Decorators # # Decorator functions without parameters are _invoked_ differently from diff --git a/scripts/makecat.py b/scripts/makecat.py index e523cff..42f4a7e 100644 --- a/scripts/makecat.py +++ b/scripts/makecat.py @@ -44,6 +44,7 @@ import codecs import pywikibot from pywikibot import date, pagegenerators, i18n, textlib +from pywikibot.tools import DequeGenerator def isdate(s): @@ -197,7 +198,7 @@ removeparent = True main = True workingcatname = '' - tocheck = [] + tocheck = DequeGenerator() for arg in pywikibot.handleArgs(): if arg.startswith('-nodate'): skipdates = True @@ -271,26 +272,14 @@ answer = workingcatname pywikibot.output(u'' + answer) pl = pywikibot.Page(mysite, answer) - tocheck = [] checked[pl] = pl include(pl) - loaded = 0 - while tocheck: - if loaded == 0: - if len(tocheck) < 50: - loaded = len(tocheck) - else: - loaded = 50 - tocheck = [x for x in pagegenerators.PreloadingGenerator(tocheck[:loaded])] - if not checkbroken: - if not tocheck[0].exists(): - pass - else: - asktoadd(tocheck[0]) - else: - asktoadd(tocheck[0]) - tocheck = tocheck[1:] - loaded -= 1 + + gen = pagegenerators.DequePreloadingGenerator(tocheck) + + for page in gen: + if checkbroken or page.exists(): + asktoadd(page) finally: try: diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 44a99cd..dd85f1b 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -11,7 +11,12 @@ import pywikibot from pywikibot import pagegenerators -from tests.aspects import unittest, TestCase, WikidataTestCase +from tests.aspects import ( + unittest, + TestCase, + WikidataTestCase, + DefaultSiteTestCase, +) class TestPageGenerators(TestCase): @@ -154,6 +159,30 @@ self.assertEqual(len(set(item['revid'] for item in items)), 4) +class TestDequePreloadingGenerator(DefaultSiteTestCase): + + """Test preloading generator on lists.""" + + def test_deque_preloading(self): + """Test pages being added to a DequePreloadingGenerator.""" + mainpage = self.get_mainpage() + + pages = pywikibot.tools.DequeGenerator([mainpage]) + gen = pagegenerators.DequePreloadingGenerator(pages) + pages_out = list() + for page in gen: + pages_out.append(page) + # Add a page to the generator + if page.namespace() == 0: + pages.extend([page.toggleTalkPage()]) + + self.assertTrue(all(isinstance(page, pywikibot.Page) for page in pages_out)) + self.assertEqual(len(pages_out), 2) + self.assertEqual(pages_out[1].namespace(), 1) + self.assertIn(mainpage, pages_out) + self.assertIn(mainpage.toggleTalkPage(), pages_out) + + class TestPreloadingItemGenerator(WikidataTestCase): """Test preloading item generator.""" -- To view, visit https://gerrit.wikimedia.org/r/169519 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088 Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Nullzero <nullzero.free(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 5 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits October 2014