[Pywikibot-commits] [Gerrit] Fix and test interwiki_graph - change (pywikibot/core)

jenkins-bot has submitted this change and it was merged. Change subject: Fix and test interwiki_graph ...................................................................... Fix and test interwiki_graph interwiki_graph used compat method Family.get_address, which doesnt exist in core and caused an exception when creating a graph. The module also relied on a 'Subject' data structure which was only defined within interwiki.py. A new simple Subject class has been added to the interwiki_graph module, and the more advanced class in interwiki.py is now a subclass. interwiki_graph.Subject provides snake_case properties replacing prior names, however the prior names can not be deprecated as the script interwiki.py uses them extensively and script_tests prevents deprecated methods from being used, as far as it can test the script entry points. Added basic dry tests for this module. Change-Id: If1f41c27a7fec012e9a7c4ce7cb13bb0ecd11c2e --- M pywikibot/family.py M pywikibot/interwiki_graph.py M scripts/interwiki.py A tests/interwiki_graph_tests.py M tests/utils.py M tox.ini 6 files changed, 203 insertions(+), 22 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/family.py b/pywikibot/family.py index b978cc6..e41017d 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -29,7 +29,7 @@ from pywikibot import config2 as config from pywikibot.tools import ( - deprecated, deprecate_arg, issue_deprecation_warning, + deprecated, deprecated_args, issue_deprecation_warning, FrozenDict, ) from pywikibot.exceptions import UnknownFamily, FamilyMaintenanceWarning @@ -885,7 +885,7 @@ return super(Family, self).__getattribute__(name) @staticmethod - @deprecate_arg('fatal', None) + @deprecated_args(fatal=None) def load(fam=None): """Import the named family. @@ -1126,6 +1126,10 @@ def rcstream_host(self, code): raise NotImplementedError("This family does not support RCStream") + @deprecated_args(name='title') + def get_address(self, code, title): + return '%s?title=%s&redirect=no' % (self.path(code), title) + def nice_get_address(self, code, title): return '%s%s' % (self.nicepath(code), title) diff --git a/pywikibot/interwiki_graph.py b/pywikibot/interwiki_graph.py index 571d5c4..bbb5896 100644 --- a/pywikibot/interwiki_graph.py +++ b/pywikibot/interwiki_graph.py @@ -1,6 +1,6 @@ """Module with the Graphviz drawing calls.""" # -# (C) Pywikibot team, 2006-2010 +# (C) Pywikibot team, 2006-2015 # # Distributed under the terms of the MIT license. # @@ -11,13 +11,17 @@ import threading -pydotfound = True try: import pydot -except ImportError: - pydotfound = False +except ImportError as e: + pydot = e + import pywikibot + from pywikibot import config2 as config + +# deprecated value +pydotfound = not isinstance(pydot, ImportError) class GraphImpossible(Exception): @@ -39,18 +43,76 @@ """ def __init__(self, graph, originPage): + """Constructor.""" threading.Thread.__init__(self) self.graph = graph self.originPage = originPage def run(self): + """Write graphs to the data directory.""" for format in config.interwiki_graph_formats: - filename = 'interwiki-graphs/' + getFilename(self.originPage, - format) + filename = config.datafilepath( + 'interwiki-graphs/' + getFilename(self.originPage, format)) if self.graph.write(filename, prog='dot', format=format): pywikibot.output(u'Graph saved as %s' % filename) else: pywikibot.output(u'Graph could not be saved as %s' % filename) + + +class Subject(object): + + """Data about a page with translations on multiple wikis.""" + + def __init__(self, origin=None): + """Constructor. + + @param originPage: the page on the 'origin' wiki + @type originPage: Page + """ + # Remember the "origin page" + self._origin = origin + + # Temporary variable to support git blame; do not use + originPage = origin + + self.found_in = None + + # foundIn is a dictionary where pages are keys and lists of + # pages are values. It stores where we found each page. + # As we haven't yet found a page that links to the origin page, we + # start with an empty list for it. + if originPage: + self.foundIn = {self.originPage: []} + else: + self.foundIn = {} + + @property + def origin(self): + """Page on the origin wiki.""" + return self._origin + + @property + def originPage(self): + """Deprecated property for the origin page. + + DEPRECATED. Use origin. + """ + # TODO: deprecate this property + return self.origin + + @property + def foundIn(self): + """Mapping of pages to others pages interwiki linked to it. + + DEPRECATED. Use found_in. + """ + # TODO: deprecate this property + return self.found_in + + @foundIn.setter + def foundIn(self, value): + """Temporary property setter to support code migration.""" + self.found_in = value class GraphDrawer: @@ -58,17 +120,25 @@ """Graphviz (dot) code creator.""" def __init__(self, subject): - """Constructor.""" - if not pydotfound: - raise GraphImpossible('pydot is not installed.') + """Constructor. + + @param subject: page data to graph + @type subject: Subject + + @raises GraphImpossible: pydot is not installed + """ + if isinstance(pydot, ImportError): + raise GraphImpossible('pydot is not installed: %s.' % pydot) self.graph = None self.subject = subject def getLabel(self, page): + """Get label for page.""" return (u'"%s:%s"' % (page.site.language(), page.title())).encode('utf-8') def addNode(self, page): + """Add a node for page.""" node = pydot.Node(self.getLabel(page), shape='rectangle') node.set_URL("\"http://%s%s\"" % (page.site.hostname(), @@ -95,6 +165,7 @@ self.graph.add_node(node) def addDirectedEdge(self, page, refPage): + """Add a directed edge from refPage to page.""" # if page was given as a hint, referrers would be [None] if refPage is not None: sourceLabel = self.getLabel(refPage) @@ -132,6 +203,7 @@ self.graph.add_edge(edge) def saveGraphFile(self): + """Write graphs to the data directory.""" thread = GraphSavingThread(self.graph, self.subject.originPage) thread.start() diff --git a/scripts/interwiki.py b/scripts/interwiki.py index 93bd89f..fbb6280 100755 --- a/scripts/interwiki.py +++ b/scripts/interwiki.py @@ -355,7 +355,9 @@ import codecs import pickle import socket + import pywikibot + from pywikibot import config, i18n, pagegenerators, textlib, interwiki_graph, titletranslate from pywikibot.tools import first_upper @@ -774,7 +776,7 @@ yield page -class Subject(object): +class Subject(interwiki_graph.Subject): u""" Class to follow the progress of a single 'subject'. @@ -844,8 +846,8 @@ if originPage: originPage = StoredPage(originPage) - # Remember the "origin page" - self.originPage = originPage + super(Subject, self).__init__(originPage) + self.repoPage = None # todo is a list of all pages that still need to be analyzed. # Mark the origin page as todo. @@ -856,14 +858,6 @@ # done is a list of all pages that have been analyzed and that # are known to belong to this subject. self.done = PageTree() - # foundIn is a dictionary where pages are keys and lists of - # pages are values. It stores where we found each page. - # As we haven't yet found a page that links to the origin page, we - # start with an empty list for it. - if originPage: - self.foundIn = {self.originPage: []} - else: - self.foundIn = {} # This is a list of all pages that are currently scheduled for # download. self.pending = PageTree() diff --git a/tests/interwiki_graph_tests.py b/tests/interwiki_graph_tests.py new file mode 100644 index 0000000..fa4d539 --- /dev/null +++ b/tests/interwiki_graph_tests.py @@ -0,0 +1,96 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Test Interwiki Graph functionality.""" +# +# (C) Pywikibot team, 2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import unicode_literals + +__version__ = '$Id$' + +from pywikibot import interwiki_graph + +from tests.aspects import unittest, SiteAttributeTestCase +from tests.utils import DryPage + + +class TestWiktionaryGraph(SiteAttributeTestCase): + + """Tests for interwiki links to local sites.""" + + sites = { + 'enwikt': { + 'family': 'wiktionary', + 'code': 'en', + }, + 'frwikt': { + 'family': 'wiktionary', + 'code': 'fr', + }, + 'plwikt': { + 'family': 'wiktionary', + 'code': 'pl', + }, + } + dry = True + + @classmethod + def setUpClass(cls): + """Setup test class.""" + if isinstance(interwiki_graph.pydot, ImportError): + raise unittest.SkipTest('pydot not installed') + super(TestWiktionaryGraph, cls).setUpClass() + + cls.pages = { + 'en': DryPage(cls.enwikt, 'origin'), + 'en2': DryPage(cls.enwikt, 'origin2'), + 'fr': DryPage(cls.frwikt, 'origin'), + 'pl': DryPage(cls.plwikt, 'origin'), + } + + def test_simple_graph(self): + """Test that GraphDrawer.createGraph does not raise exception.""" + data = interwiki_graph.Subject(self.pages['en']) + + data.found_in[self.pages['en']] = [self.pages['fr'], self.pages['pl']] + data.found_in[self.pages['fr']] = [self.pages['en'], self.pages['pl']] + data.found_in[self.pages['pl']] = [self.pages['en'], self.pages['fr']] + + drawer = interwiki_graph.GraphDrawer(data) + + drawer.createGraph() + + def test_octagon(self): + """Test octagon nodes.""" + data = interwiki_graph.Subject(self.pages['en']) + + data.found_in[self.pages['en']] = [self.pages['fr'], self.pages['pl']] + data.found_in[self.pages['en2']] = [self.pages['fr']] + data.found_in[self.pages['fr']] = [self.pages['en'], self.pages['pl']] + data.found_in[self.pages['pl']] = [self.pages['en'], self.pages['fr']] + + drawer = interwiki_graph.GraphDrawer(data) + + drawer.createGraph() + + nodes = drawer.graph.obj_dict['nodes'] + self.assertEqual( + nodes['"pl:origin"'][0]['attributes']['shape'], + 'rectangle') + + self.assertEqual( + nodes['"fr:origin"'][0]['attributes']['shape'], + 'rectangle') + + self.assertEqual( + nodes['"en:origin"'][0]['attributes']['shape'], + 'octagon') + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass diff --git a/tests/utils.py b/tests/utils.py index 78645ac..05bed11 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -259,6 +259,19 @@ }) +class DryPage(pywikibot.Page): + + """Dummy class that acts like a Page but avoids network activity.""" + + _pageid = 1 + _disambig = False + _isredir = False + + def isDisambig(self): + """Return disambig status stored in _disambig.""" + return self._disambig + + def execute(command, data_in=None, timeout=0, error=None): """ Execute a command and capture outputs. diff --git a/tox.ini b/tox.ini index 1642ed7..e34218b 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,7 @@ pywikibot/echo.py \ pywikibot/editor.py \ pywikibot/exceptions.py \ + pywikibot/interwiki_graph.py \ pywikibot/families/ \ pywikibot/fixes.py \ pywikibot/i18n.py \ @@ -117,6 +118,7 @@ tests/i18n/ \ tests/l10n_tests.py \ tests/link_tests.py \ + tests/interwiki_graph_tests.py \ tests/ipregex_tests.py \ tests/pwb/ \ tests/pwb_tests.py \ -- To view, visit https://gerrit.wikimedia.org/r/186618 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If1f41c27a7fec012e9a7c4ce7cb13bb0ecd11c2e Gerrit-PatchSet: 14 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Ladsgroup &lt;ladsgroup(a)gmail.com&gt; Gerrit-Reviewer: Merlijn van Deen &lt;valhallasw(a)arctus.nl&gt; Gerrit-Reviewer: XZise &lt;CommodoreFabianus(a)gmx.de&gt; Gerrit-Reviewer: jenkins-bot <>