jenkins-bot has submitted this change and it was merged.
Change subject: Load Flow object content in Pywikibot ......................................................................
Load Flow object content in Pywikibot
This change allows Pywikibot bots to load basic Flow content from Flow-enabled wikis. This includes loading the contents of posts, loading topics from boards, and loading replies to topics and posts. Tests have been added to ensure that content is indeed loaded.
Bug: T101260 Bug: T101261 Bug: T101262 Bug: T101263 Change-Id: Ibd84d0763a7745da6410c194b8d5ddf9ba8b57de --- M pywikibot/flow.py M pywikibot/site.py M tests/flow_tests.py 3 files changed, 429 insertions(+), 31 deletions(-)
Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/flow.py b/pywikibot/flow.py index 8645532..c9fe320 100644 --- a/pywikibot/flow.py +++ b/pywikibot/flow.py @@ -11,7 +11,16 @@
import logging
+from pywikibot.exceptions import NoPage, UnknownExtension from pywikibot.page import BasePage +from pywikibot.tools import PY2 + +if not PY2: + unicode = str + basestring = (str,) + from urllib.parse import urlparse, parse_qs +else: + from urlparse import urlparse, parse_qs
logger = logging.getLogger('pywiki.wiki.flow') @@ -43,7 +52,7 @@ super(FlowPage, self).__init__(source, title)
if not self.site.has_extension('Flow'): - raise ValueError('site is not Flow-enabled') + raise UnknownExtension('site is not Flow-enabled')
def _load_uuid(self): """Load and save the UUID of the page.""" @@ -60,6 +69,13 @@ self._load_uuid() return self._uuid
+ def get(self, force=False, get_redirect=False, sysop=False): + if get_redirect or force or sysop: + raise NotImplementedError + + # TODO: Return more useful data + return self._data +
class Board(FlowPage):
@@ -70,6 +86,57 @@ if not hasattr(self, '_data'): self._data = self.site.load_board(self) return self._data + + def _parse_url(self, links): + """Parse a URL retrieved from the API.""" + rule = links['fwd'] + parsed_url = urlparse(rule['url']) + params = parse_qs(parsed_url.query) + new_params = {} + for key, value in params.items(): + if key != 'title': + key = key.replace('topiclist_', '').replace('-', '_') + if key == 'offset_dir': + new_params['reverse'] = (value == 'rev') + else: + new_params[key] = value + return new_params + + def topics(self, format='wikitext', limit=100, sort_by='newest', + offset=None, offset_uuid='', reverse=False, + include_offset=False, toc_only=False): + """Load this board's topics. + + @param format: The content format to request the data in. + @type format: str (either 'wikitext', 'html', or 'fixed-html') + @param limit: The number of topics to fetch in each request. + @type limit: int + @param sort_by: Algorithm to sort topics by. + @type sort_by: str (either 'newest' or 'updated') + @param offset: The timestamp to start at (when sortby is 'updated'). + @type offset: Timestamp or equivalent str + @param offset_uuid: The UUID to start at (when sortby is 'newest'). + @type offset_uuid: str (in the form of a UUID) + @param reverse: Whether to reverse the topic ordering. + @type reverse: bool + @param include_offset: Whether to include the offset topic. + @type include_offset: bool + @param toc_only: Whether to only include information for the TOC. + @type toc_only: bool + @return: A generator of this board's topics. + @rtype: generator of Topic objects + """ + data = self.site.load_topiclist(self, format=format, limit=limit, + sortby=sort_by, toconly=toc_only, + offset=offset, offset_id=offset_uuid, + reverse=reverse, + include_offset=include_offset) + while data['roots']: + for root in data['roots']: + topic = Topic.from_topiclist_data(self, root, data) + yield topic + cont_args = self._parse_url(data['links']['pagination']) + data = self.site.load_topiclist(self, **cont_args)
class Topic(FlowPage): @@ -82,8 +149,52 @@ self._data = self.site.load_topic(self) return self._data
+ @classmethod + def from_topiclist_data(cls, board, root_uuid, topiclist_data): + """Create a Topic object from API data.
-# Flow non-page-like objects (currently just posts) + @param board: The topic's parent Flow board + @type board: Board + @param root_uuid: The UUID of the topic and its root post + @type root_uuid: unicode + @param topiclist_data: The data returned by view-topiclist + @type topiclist_data: dict + @return: A Topic object derived from the supplied data + @rtype: Topic + @raise TypeError: any passed parameters have wrong types + @raise ValueError: the passed topiclist_data is missing required data + """ + if not isinstance(board, Board): + raise TypeError('board must be a pywikibot.flow.Board object.') + if not isinstance(root_uuid, basestring): + raise TypeError('Topic/root UUID must be a string.') + + topic = cls(board.site, 'Topic:' + root_uuid) + topic._root = Post.fromJSON(topic, root_uuid, topiclist_data) + topic._uuid = root_uuid + return topic + + @property + def root(self): + """The root post of this topic.""" + if not hasattr(self, '_root'): + self._root = Post.fromJSON(self, self.uuid, self._data) + return self._root + + def replies(self, format='wikitext', force=False): + """A list of replies to this topic's root post. + + @param format: Content format to return contents in + @type format: str ('wikitext', 'html', or 'fixed-html') + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @return: The replies of this topic's root post + @rtype: list of Posts + """ + return self.root.replies(format=format, force=force) + + +# Flow non-page-like objects class Post(object):
"""A post to a Flow discussion topic.""" @@ -92,22 +203,81 @@ """ Constructor.
- @param page: Flow topic or board - @type page: FlowPage + @param page: Flow topic + @type page: Topic @param uuid: UUID of a Flow post @type uuid: unicode
@raise TypeError: incorrect types of parameters - @raise ValueError: use of non-Flow-enabled Site or invalid UUID """ - if not isinstance(page, FlowPage): - raise TypeError('page must be a FlowPage object') - - if not uuid: - raise ValueError('post UUID must be provided') + if not isinstance(page, Topic): + raise TypeError('Page must be a Topic object') + if not page.exists(): + raise NoPage(page, 'Topic must exist: %s') + if not isinstance(uuid, basestring): + raise TypeError('Post UUID must be a string')
self._page = page self._uuid = uuid + + self._content = {} + + @classmethod + def fromJSON(cls, page, post_uuid, data): + """ + Create a Post object using the data returned from the API call. + + @param page: A Flow topic + @type page: Topic + @param post_uuid: The UUID of the post + @type post_uuid: unicode + @param data: The JSON data returned from the API + @type data: dict + + @return: A Post object + @raise TypeError: data is not a dict + @raise ValueError: data is missing required entries + """ + post = cls(page, post_uuid) + post._set_data(data) + + return post + + def _set_data(self, data): + """Set internal data and cache content. + + @param data: The data to store internally + @type data: dict + @raise TypeError: data is not a dict + @raise ValueError: missing data entries or post/revision not found + """ + if not isinstance(data, dict): + raise TypeError('Illegal post data (must be a dictionary).') + if ('posts' not in data) or ('revisions' not in data): + raise ValueError('Illegal post data (missing required data).') + if self.uuid not in data['posts']: + raise ValueError('Post not found in supplied data.') + + self._data = data + current_revision_id = data['posts'][self.uuid][0] + + if current_revision_id not in data['revisions']: + raise ValueError('Current revision of post' + 'not found in supplied data.') + + self._current_revision = data['revisions'][current_revision_id] + if 'content' in self._current_revision: + content = self._current_revision.pop('content') + assert isinstance(content, dict) + assert isinstance(content['content'], unicode) + self._content[content['format']] = content['content'] + + def _load(self, format='wikitext'): + """Load and cache the Post's data using the given content format.""" + data = self.site.load_post_current_revision(self.page, self.uuid, + format) + self._set_data(data) + return self._data
@property def uuid(self): @@ -135,3 +305,47 @@ @rtype: FlowPage """ return self._page + + def get(self, format='wikitext', force=False, sysop=False): + """Return the contents of the post in the given format. + + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @param sysop: Whether to load using sysop rights. Implies force. + @type sysop: bool + @param format: Content format to return contents in + @type format: unicode + @return: The contents of the post in the given content format + @rtype: unicode + @raise NotImplementedError: use of 'sysop' + """ + if sysop: + raise NotImplementedError + + if format not in self._content or force: + self._load(format) + return self._content[format] + + def replies(self, format='wikitext', force=False): + """Return this post's replies. + + @param format: Content format to return contents in + @type format: str ('wikitext', 'html', or 'fixed-html') + @param force: Whether to reload from the API instead of using the cache + @type force: bool + @return This post's replies + @rtype: list of Posts + """ + if format not in ('wikitext', 'html', 'fixed-html'): + raise ValueError('Invalid content format.') + + if hasattr(self, '_replies') and not force: + return self._replies + + if not hasattr(self, '_current_revision') or force: + self._load(format) + + reply_uuids = self._current_revision['replies'] + self._replies = [Post(self.page, uuid) for uuid in reply_uuids] + + return self._replies diff --git a/pywikibot/site.py b/pywikibot/site.py index 8208363..5ba136e 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -5771,11 +5771,52 @@
@param page: A Flow board @type page: Board - @return: A dict representing the board's data. + @return: A dict representing the board's metadata. @rtype: dict """ req = self._simple_request(action='flow', page=page, - submodule='view-topiclist') + submodule='view-topiclist', + vtllimit=1) + data = req.submit() + return data['flow']['view-topiclist']['result']['topiclist'] + + @need_extension('Flow') + def load_topiclist(self, page, format='wikitext', limit=100, + sortby='newest', toconly=False, offset=None, + offset_id=None, reverse=False, include_offset=False): + """Retrieve the topiclist of a Flow board. + + @param page: A Flow board + @type page: Board + @param format: The content format to request the data in. + @type format: str (either 'wikitext', 'html', or 'fixed-html') + @param limit: The number of topics to fetch in each request. + @type limit: int + @param sortby: Algorithm to sort topics by. + @type sortby: str (either 'newest' or 'updated') + @param toconly: Whether to only include information for the TOC. + @type toconly: bool + @param offset: The timestamp to start at (when sortby is 'updated'). + @type offset: Timestamp or equivalent str + @param offset_id: The topic UUID to start at (when sortby is 'newest'). + @type offset_id: str (in the form of a UUID) + @param reverse: Whether to reverse the topic ordering. + @type reverse: bool + @param include_offset: Whether to include the offset topic. + @type include_offset: bool + @return: A dict representing the board's topiclist. + @rtype: dict + """ + if offset: + offset = pywikibot.Timestamp.fromtimestampformat(offset) + offset_dir = reverse and 'rev' or 'fwd' + + params = {'action': 'flow', 'submodule': 'view-topiclist', 'page': page, + 'vtlformat': format, 'vtlsortby': sortby, + 'vtllimit': limit, 'vtloffset-dir': offset_dir, + 'vtloffset': offset, 'vtloffset-id': offset_id, + 'vtlinclude-offset': include_offset, 'vtltoconly': toconly} + req = self._request(parameters=params) data = req.submit() return data['flow']['view-topiclist']['result']['topiclist']
@@ -5793,6 +5834,25 @@ data = req.submit() return data['flow']['view-topic']['result']['topic']
+ @need_extension('Flow') + def load_post_current_revision(self, page, post_id, format): + """Retrieve the data for a post to a Flow topic. + + @param page: A Flow topic + @type page: Topic + @param post_id: The UUID of the Post + @type post_id: unicode + @param format: The content format used for the returned content + @type format: unicode (either 'wikitext', 'html', or 'fixed-html') + @return: A dict representing the post data for the given UUID. + @rtype: dict + """ + req = self._simple_request(action='flow', page=page, + submodule='view-post', vppostId=post_id, + vpformat=format) + data = req.submit() + return data['flow']['view-post']['result']['topic'] + def watched_pages(self, sysop=False, force=False, step=None, total=None): """ Return watchlist. diff --git a/tests/flow_tests.py b/tests/flow_tests.py index 8c4f85d..5ba6b7b 100644 --- a/tests/flow_tests.py +++ b/tests/flow_tests.py @@ -9,8 +9,9 @@
__version__ = '$Id$'
-import pywikibot -import pywikibot.flow +from pywikibot.exceptions import NoPage +from pywikibot.flow import Board, Topic, Post +from pywikibot.tools import PY2
from tests.aspects import ( TestCase, @@ -21,29 +22,55 @@ BasePageLoadRevisionsCachingTestBase, )
+if not PY2: + unicode = str +
class TestBoardBasePageMethods(BasePageMethodsTestBase):
- """Test Flow pages using BasePage-defined methods.""" + """Test Flow board pages using BasePage-defined methods."""
family = 'mediawiki' code = 'mediawiki'
def setUp(self): """Set up unit test.""" - self._page = pywikibot.flow.Board( - self.site, 'Talk:Sandbox') + self._page = Board(self.site, 'Talk:Sandbox') super(TestBoardBasePageMethods, self).setUp()
def test_basepage_methods(self): - """Test basic Page methods on a Flow page.""" + """Test basic Page methods on a Flow board page.""" self._test_invoke() self._test_return_datatypes() - self.assertEqual(self._page.isRedirectPage(), False) + self.assertFalse(self._page.isRedirectPage()) self.assertEqual(self._page.latest_revision.parent_id, 0)
def test_content_model(self): """Test Flow page content model.""" + self.assertEqual(self._page.content_model, 'flow-board') + + +class TestTopicBasePageMethods(BasePageMethodsTestBase): + + """Test Flow topic pages using BasePage-defined methods.""" + + family = 'mediawiki' + code = 'mediawiki' + + def setUp(self): + """Set up unit test.""" + self._page = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + super(TestTopicBasePageMethods, self).setUp() + + def test_basepage_methods(self): + """Test basic Page methods on a Flow topic page.""" + self._test_invoke() + self._test_return_datatypes() + self.assertFalse(self._page.isRedirectPage()) + self.assertEqual(self._page.latest_revision.parent_id, 0) + + def test_content_model(self): + """Test Flow topic page content model.""" self.assertEqual(self._page.content_model, 'flow-board')
@@ -56,12 +83,12 @@
def setUp(self): """Set up unit test.""" - self._page = pywikibot.flow.Board( - self.site, 'Talk:Sandbox') + self._page = Board(self.site, 'Talk:Sandbox') super(TestLoadRevisionsCaching, self).setUp()
def test_page_text(self): """Test site.loadrevisions() with Page.text.""" + self.skipTest('See T107537') self._test_page_text()
@@ -76,15 +103,13 @@
def test_board_uuid(self): """Test retrieval of Flow board UUID.""" - site = self.get_site() - board = pywikibot.flow.Board(site, u'Talk:Sandbox') - self.assertEqual(board.uuid, u'rl7iby6wgksbpfno') + board = Board(self.site, 'Talk:Sandbox') + self.assertEqual(board.uuid, 'rl7iby6wgksbpfno')
def test_topic_uuid(self): """Test retrieval of Flow topic UUID.""" - site = self.get_site() - topic = pywikibot.flow.Topic(site, u'Topic:Sh6wgo5tu3qui1w2') - self.assertEqual(topic.uuid, u'sh6wgo5tu3qui1w2') + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + self.assertEqual(topic.uuid, 'sh6wgo5tu3qui1w2')
def test_post_uuid(self): """Test retrieval of Flow post UUID. @@ -93,7 +118,106 @@ the property to make sure the UUID passed to the constructor is stored properly. """ - site = self.get_site() - topic = pywikibot.flow.Topic(site, u'Topic:Sh6wgo5tu3qui1w2') - post = pywikibot.flow.Post(topic, u'sh6wgoagna97q0ia') - self.assertEqual(post.uuid, u'sh6wgoagna97q0ia') + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + post = Post(topic, 'sh6wgoagna97q0ia') + self.assertEqual(post.uuid, 'sh6wgoagna97q0ia') + + def test_post_contents(self): + """Test retrieval of Flow post contents.""" + # Load + topic = Topic(self.site, 'Topic:Sh6wgo5tu3qui1w2') + post = Post(topic, 'sh6wgoagna97q0ia') + # Wikitext + wikitext = post.get(format='wikitext') + self.assertIn('wikitext', post._content) + self.assertNotIn('html', post._content) + self.assertIsInstance(wikitext, unicode) + self.assertNotEqual(wikitext, '') + # HTML + html = post.get(format='html') + self.assertIn('html', post._content) + self.assertIn('wikitext', post._content) + self.assertIsInstance(html, unicode) + self.assertNotEqual(html, '') + # Caching (hit) + post._content['html'] = 'something' + html = post.get(format='html') + self.assertIsInstance(html, unicode) + self.assertEqual(html, 'something') + self.assertIn('html', post._content) + # Caching (reload) + post._content['html'] = 'something' + html = post.get(format='html', force=True) + self.assertIsInstance(html, unicode) + self.assertNotEqual(html, 'something') + self.assertIn('html', post._content) + + def test_topiclist(self): + """Test loading of topiclist.""" + board = Board(self.site, 'Talk:Sandbox') + i = 0 + for topic in board.topics(limit=7): + i += 1 + if i == 10: + break + self.assertEqual(i, 10) + + +class TestFlowFactoryErrors(TestCase): + + """Test errors associated with class methods generating Flow objects.""" + + family = 'test' + code = 'test' + + cached = True + + def test_illegal_arguments(self): + """Test illegal method arguments.""" + board = Board(self.site, 'Talk:Pywikibot test') + real_topic = Topic(self.site, 'Topic:Slbktgav46omarsd') + fake_topic = Topic(self.site, 'Topic:Abcdefgh12345678') + # Topic.from_topiclist_data + self.assertRaises(TypeError, Topic.from_topiclist_data, self.site, '', {}) + self.assertRaises(TypeError, Topic.from_topiclist_data, board, 521, {}) + self.assertRaises(TypeError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', [0, 1, 2]) + self.assertRaises(NoPage, Topic.from_topiclist_data, board, + 'abc', {'stuff': 'blah'}) + + # Post.fromJSON + self.assertRaises(TypeError, Post.fromJSON, board, 'abc', {}) + self.assertRaises(TypeError, Post.fromJSON, real_topic, 1234, {}) + self.assertRaises(TypeError, Post.fromJSON, real_topic, 'abc', []) + self.assertRaises(NoPage, Post.fromJSON, fake_topic, 'abc', + {'posts': [], 'revisions': []}) + + def test_invalid_data(self): + """Test invalid "API" data.""" + board = Board(self.site, 'Talk:Pywikibot test') + real_topic = Topic(self.site, 'Topic:Slbktgav46omarsd') + # Topic.from_topiclist_data + self.assertRaises(ValueError, Topic.from_topiclist_data, + board, 'slbktgav46omarsd', {'stuff': 'blah'}) + self.assertRaises(ValueError, Topic.from_topiclist_data, + board, 'slbktgav46omarsd', + {'posts': [], 'revisions': []}) + self.assertRaises(ValueError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', + {'posts': {'slbktgav46omarsd': ['123']}, + 'revisions': {'456': []}}) + self.assertRaises(AssertionError, Topic.from_topiclist_data, board, + 'slbktgav46omarsd', + {'posts': {'slbktgav46omarsd': ['123']}, + 'revisions': {'123': {'content': 789}}}) + + # Post.fromJSON + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', {}) + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', + {'stuff': 'blah'}) + self.assertRaises(ValueError, Post.fromJSON, real_topic, 'abc', + {'posts': {'abc': ['123']}, + 'revisions': {'456': []}}) + self.assertRaises(AssertionError, Post.fromJSON, real_topic, 'abc', + {'posts': {'abc': ['123']}, + 'revisions': {'123': {'content': 789}}})