Revision: 6469
Author: russblau
Date: 2009-03-02 15:30:52 +0000 (Mon, 02 Mar 2009)
Log Message:
-----------
fix bug that was trying to read generator's entire contents before preloading any pages
Modified Paths:
--------------
branches/rewrite/pywikibot/pagegenerators.py
Modified: branches/rewrite/pywikibot/pagegenerators.py
===================================================================
--- branches/rewrite/pywikibot/pagegenerators.py 2009-03-02 15:30:23 UTC (rev 6468)
+++ branches/rewrite/pywikibot/pagegenerators.py 2009-03-02 15:30:52 UTC (rev 6469)
@@ -655,11 +655,18 @@
sites = {}
# build a list of pages for each site found in the iterator
for page in generator:
- sites.setdefault(page.site(), []).append(page)
- return itertools.chain(*(site.preloadpages(sites[site], pageNumber)
- for site in sites))
+ site = page.site()
+ sites.setdefault(site, []).append(page)
+ if len(sites[site]) >= pageNumber:
+ group = sites[site]
+ sites[site] = []
+ for i in site.preloadpages(group, pageNumber):
+ yield i
+ for site in sites:
+ if sites[site]:
+ for i in site.preloadpages(sites[site], pageNumber):
+ yield i
-
#TODO below
def UnusedFilesGenerator(number=100, repeat=False, site=None, extension=None):
Revision: 6466
Author: nicdumz
Date: 2009-03-02 13:57:14 +0000 (Mon, 02 Mar 2009)
Log Message:
-----------
This was meant to be in r6461. The redirect target is not necessary here.
Modified Paths:
--------------
branches/rewrite/pywikibot/scripts/category.py
Modified: branches/rewrite/pywikibot/scripts/category.py
===================================================================
--- branches/rewrite/pywikibot/scripts/category.py 2009-03-02 11:47:41 UTC (rev 6465)
+++ branches/rewrite/pywikibot/scripts/category.py 2009-03-02 13:57:14 UTC (rev 6466)
@@ -374,11 +374,9 @@
pywikibot.output(u"%s doesn't exist yet. Ignoring."
% (page.title()))
pass
- except pywikibot.IsRedirectPage, arg:
- redirTarget = pywikibot.Page(site, arg.args[0])
+ except pywikibot.IsRedirectPage:
pywikibot.output(
- u"WARNING: %s is redirect to %s. Ignoring."
- % (page.title(), redirTarget.title()))
+ u"WARNING: %s is redirect to. Ignoring." % page)
else:
cats = page.categories()
# Show the title of the page we're working on.
Revision: 6461
Author: nicdumz
Date: 2009-03-02 07:00:46 +0000 (Mon, 02 Mar 2009)
Log Message:
-----------
Introducing a PageRelatedError abstract class, to use class level messages to have simpler Error creations, and unified Error message.
For now only NoPage, IsRedirectPage, and IsNotRedirectPage are subclasses of that abstract Error: others could too, I'm just converting the Errors that we will keep _for sure_
Modified Paths:
--------------
branches/rewrite/pywikibot/exceptions.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/site.py
Modified: branches/rewrite/pywikibot/exceptions.py
===================================================================
--- branches/rewrite/pywikibot/exceptions.py 2009-03-01 09:19:24 UTC (rev 6460)
+++ branches/rewrite/pywikibot/exceptions.py 2009-03-02 07:00:46 UTC (rev 6461)
@@ -25,24 +25,47 @@
def __str__(self):
return self.string
+class PageRelatedError(Error):
+ """Abstract Exception, used when the Exception concerns a particular
+ Page, and when a generic message can be written once for all"""
+ # Preformated UNICODE message where the page title will be inserted
+ # Override this in subclasses.
+ # u"Oh noes! Page %s is too funky, we should not delete it ;("
+ message = None
+ def __init__(self, page):
+ """
+ @param page
+ @type page: Page object
+ """
+ if self.message is None:
+ raise Error("PageRelatedError is abstract. Can't instantiate it!")
+ super(PageRelatedError, self).__init__(self.message % page)
+ self._page = page
+
+ def getPage(self):
+ return self._page
+
class NoUsername(Error):
"""Username is not in user-config.py"""
-class NoPage(Error):
+class NoPage(PageRelatedError):
"""Page does not exist"""
+ message = u"Page %s doesn't exist."
class NoSuchSite(Error):
"""Site does not exist"""
-class IsRedirectPage(Error):
+class IsRedirectPage(PageRelatedError):
"""Page is a redirect page"""
+ message = u"Page %s is a redirect page."
-class IsNotRedirectPage(Error):
+class IsNotRedirectPage(PageRelatedError):
"""Page is not a redirect page"""
+ message = u"Page %s is not a redirect page."
class CircularRedirect(Error):
"""Page is a circular redirect
-
+
Exception argument is the redirect target; this may be the same title
as this page or a different title (in which case the target page directly
or indirectly redirects back to this one)
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2009-03-01 09:19:24 UTC (rev 6460)
+++ branches/rewrite/pywikibot/page.py 2009-03-02 07:00:46 UTC (rev 6461)
@@ -283,8 +283,7 @@
exceptions that should be caught by the calling code:
- NoPage: The page does not exist
- - IsRedirectPage: The page is a redirect. The argument of the
- exception is the title of the page it redirects to.
+ - IsRedirectPage: The page is a redirect.
- SectionError: The section does not exist on a page with a #
link
@@ -304,7 +303,7 @@
else:
# Make sure we re-raise an exception we got on an earlier attempt
if hasattr(self, '_redirarg') and not get_redirect:
- raise pywikibot.IsRedirectPage, self._redirarg
+ raise pywikibot.IsRedirectPage(self)
elif hasattr(self, '_getexception'):
raise self._getexception
if force or not hasattr(self, "_revid") \
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2009-03-01 09:19:24 UTC (rev 6460)
+++ branches/rewrite/pywikibot/site.py 2009-03-02 07:00:46 UTC (rev 6461)
@@ -939,7 +939,7 @@
def page_restrictions(self, page):
"""Returns a dictionary reflecting page protections"""
if not self.page_exists(page):
- raise NoPage(u'No page %s.' % page)
+ raise NoPage(page)
if not hasattr(page, "_protection"):
self.loadpageinfo(page)
return page._protection
@@ -970,7 +970,7 @@
if not hasattr(page, "_redir"):
self.loadpageinfo(page)
if not page._redir:
- raise pywikibot.IsNotRedirectPage(page.title())
+ raise pywikibot.IsNotRedirectPage(page)
title = page.title(withSection=False)
query = api.Request(site=self, action="query", property="info",
inprop="protection|talkid|subjectid",
@@ -1348,8 +1348,7 @@
u"loadrevisions: Query on %s returned data on '%s'"
% (page, pagedata['title']))
if "missing" in pagedata:
- raise NoPage(u'Page %s does not exist'
- % page.title(asLink=True))
+ raise NoPage(page)
else:
page = Page(self, pagedata['title'])
api.update_page(page, pagedata)
@@ -2141,8 +2140,7 @@
except NoPage:
lastrev = None
if not recreate:
- raise Error("Page %s does not exist on %s wiki."
- % (page.title(withSection=False), self))
+ raise
token = self.token(page, "edit")
self.lock_page(page)
if lastrev is not None and page.latestRevision() != lastrev:
Revision: 6463
Author: nicdumz
Date: 2009-03-02 08:42:35 +0000 (Mon, 02 Mar 2009)
Log Message:
-----------
Any reason not to use config.console_encoding ?
Using sys.std(out|err).encoding still causes problems when output is piped. Depending on how the next process handle the pipe, .encoding can be None.
If there's a particular reason for these to use sys.stderr.encoding, that'd be nice to use a config variable, and test for its non-"Noneness", as we currently do for config.console_encoding
Modified Paths:
--------------
branches/rewrite/pywikibot/exceptions.py
branches/rewrite/pywikibot/page.py
Modified: branches/rewrite/pywikibot/exceptions.py
===================================================================
--- branches/rewrite/pywikibot/exceptions.py 2009-03-02 08:20:02 UTC (rev 6462)
+++ branches/rewrite/pywikibot/exceptions.py 2009-03-02 08:42:35 UTC (rev 6463)
@@ -10,7 +10,7 @@
__version__ = '$Id$'
-import sys
+from pywikibot import config
# TODO: These are copied from wikipedia.py; not certain that all of them
# will be needed in the rewrite.
@@ -19,7 +19,7 @@
"""Wikipedia error"""
def __init__(self, arg):
try:
- self.string = arg.encode(sys.stderr.encoding, "xmlcharrefreplace")
+ self.string = arg.encode(config.console_encoding, "xmlcharrefreplace")
except (AttributeError, TypeError):
self.string = arg.encode("ascii", "xmlcharrefreplace")
def __str__(self):
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2009-03-02 08:20:02 UTC (rev 6462)
+++ branches/rewrite/pywikibot/page.py 2009-03-02 08:42:35 UTC (rev 6463)
@@ -18,7 +18,6 @@
import htmlentitydefs
import logging
import re
-import sys
import threading
import unicodedata
import urllib
@@ -214,7 +213,7 @@
def __str__(self):
"""Return a console representation of the pagelink."""
return self.title(asLink=True, forceInterwiki=True
- ).encode(sys.stderr.encoding)
+ ).encode(config.console_encoding)
def __unicode__(self):
return self.title(asLink=True, forceInterwiki=True)
@@ -222,7 +221,7 @@
def __repr__(self):
"""Return a more complete string representation."""
return u"%s(%s)" % (self.__class__.__name__,
- self.title().encode(sys.stderr.encoding))
+ self.title().encode(config.console_encoding))
def __cmp__(self, other):
"""Test for equality and inequality of Page objects.
Revision: 6462
Author: nicdumz
Date: 2009-03-02 08:20:02 +0000 (Mon, 02 Mar 2009)
Log Message:
-----------
* Splitting Page._redir into Page._isredir and Page._redirtarget:
** Filling ._isredir with a (boolean) value can be done easily in update_page
** But getting the redirect target requires an additional API request
** And mixing both usages is not necessary! Knowing that we have a redirect page without knowing its target is sometimes enough.
* Avoiding code duplication in Page.isRedirect, Page.getRediretTarget, Site.getredirtarget, and Site.page_isredirect
* Implementing a clean Page.get():
** adding error handling
** adding a naive Page._getInternals to simplify code
Modified Paths:
--------------
branches/rewrite/pywikibot/data/api.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/site.py
Modified: branches/rewrite/pywikibot/data/api.py
===================================================================
--- branches/rewrite/pywikibot/data/api.py 2009-03-02 07:00:46 UTC (rev 6461)
+++ branches/rewrite/pywikibot/data/api.py 2009-03-02 08:20:02 UTC (rev 6462)
@@ -648,7 +648,7 @@
raise AssertionError(
"Page %s has neither 'pageid' nor 'missing' attribute"
% pagedict['title'])
- page._redir = 'redirect' in pagedict
+ page._isredir = 'redirect' in pagedict
if 'touched' in pagedict:
page._timestamp = pagedict['touched']
if 'protection' in pagedict:
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2009-03-02 07:00:46 UTC (rev 6461)
+++ branches/rewrite/pywikibot/page.py 2009-03-02 08:20:02 UTC (rev 6462)
@@ -295,24 +295,45 @@
"""
if force:
- # When forcing, we retry the page no matter what. Old exceptions
- # do not apply any more.
- for attr in ['_redirarg', '_getexception']:
+ # When forcing, we retry the page no matter what:
+ # * Old exceptions do not apply any more
+ # * Deleting _revid to force reload
+ # * Deleting _redirtarget, that info is now obsolete.
+ for attr in ['_redirtarget', '_getexception', '_revid']:
if hasattr(self, attr):
delattr(self,attr)
- else:
- # Make sure we re-raise an exception we got on an earlier attempt
- if hasattr(self, '_redirarg') and not get_redirect:
- raise pywikibot.IsRedirectPage(self)
- elif hasattr(self, '_getexception'):
- raise self._getexception
- if force or not hasattr(self, "_revid") \
+ try:
+ self._getInternals(sysop)
+ except pywikibot.IsRedirectPage:
+ if not get_redirect:
+ raise
+
+ return self._revisions[self._revid].text
+
+ def _getInternals(self, sysop):
+ """Helper function for get().
+ Stores latest revision in self if it doesn't contain it, doesn't think.
+ * Raises exceptions from previous runs.
+ * Stores new exceptions in _getexception and raises them"""
+
+ # Raise exceptions from previous runs
+ if hasattr(self, '_getexception'):
+ raise self._getexception
+
+ # If not already stored, fetch revision
+ if not hasattr(self, "_revid") \
or not self._revid in self._revisions \
or self._revisions[self._revid].text is None:
- self.site().loadrevisions(self, getText=True, sysop=sysop)
- # TODO: Exception handling for no-page, redirects, etc.
+ try:
+ self.site().loadrevisions(self, getText=True, sysop=sysop)
+ except (pywikibot.NoPage, pywikibot.SectionError), e:
+ self._getexception = e
+ raise
- return self._revisions[self._revid].text
+ # self._isredir is set by loadrevisions
+ if self._isredir:
+ self._getexception = pywikibot.IsRedirectPage(self)
+ raise self._getexception
@deprecate_arg("throttle", None)
@deprecate_arg("nofollow_redirects", None)
@@ -864,11 +885,7 @@
exception. This method also can raise a NoPage exception.
"""
- if not self.isRedirectPage():
- raise pywikibot.IsNotRedirectPage(self.title())
- if not isinstance(self._redir, Page):
- self.site().getredirtarget(self)
- return self._redir
+ return self.site().getredirtarget(self)
@deprecate_arg("forceReload", None)
def getVersionHistory(self, reverseOrder=False, getAll=False,
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2009-03-02 07:00:46 UTC (rev 6461)
+++ branches/rewrite/pywikibot/site.py 2009-03-02 08:20:02 UTC (rev 6462)
@@ -961,16 +961,16 @@
def page_isredirect(self, page):
"""Return True if and only if page is a redirect."""
- if not hasattr(page, "_redir"):
+ if not hasattr(page, "_isredir"):
self.loadpageinfo(page)
- return bool(page._redir)
+ return page._isredir
def getredirtarget(self, page):
"""Return Page object for the redirect target of page."""
- if not hasattr(page, "_redir"):
- self.loadpageinfo(page)
- if not page._redir:
+ if not self.page_isredirect(page):
raise pywikibot.IsNotRedirectPage(page)
+ if hasattr(page, '_redirtarget'):
+ return page._redirtarget
title = page.title(withSection=False)
query = api.Request(site=self, action="query", property="info",
inprop="protection|talkid|subjectid",
@@ -998,7 +998,8 @@
% pagedata['title'])
target = pywikibot.Page(self, pagedata['title'], pagedata['ns'])
api.update_page(target, pagedata)
- page._redir = target
+ page._redirtarget = target
+ return page._redirtarget
def preloadpages(self, pagelist, groupsize=50):
"""Return a generator to a list of preloaded pages.
Revision: 6460
Author: nicdumz
Date: 2009-03-01 09:19:24 +0000 (Sun, 01 Mar 2009)
Log Message:
-----------
* Bugfix for Page.delete : Site.delete doesn't exist -> Site.deletepage
* Tweaking error handling in delete methods to catch NoUsername exceptions in Page and not in Site, to be able to implement mark support in Page itself.
Modified Paths:
--------------
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/site.py
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2009-03-01 09:16:30 UTC (rev 6459)
+++ branches/rewrite/pywikibot/page.py 2009-03-01 09:19:24 UTC (rev 6460)
@@ -963,6 +963,7 @@
speedy-deletion request on the page instead.
"""
+ # TODO: add support for mark
if reason is None:
pywikibot.output(u'Deleting %s.' % (self.title(asLink=True)))
reason = pywikibot.input(u'Please enter a reason for the deletion:')
@@ -977,7 +978,13 @@
answer = 'y'
self.site()._noDeletePrompt = True
if answer in ['y', 'Y']:
- return self.site().delete(self, reason, mark=mark)
+ try:
+ return self.site().deletepage(self, reason)
+ except pywikibot.NoUsername, e:
+ if mark:
+ raise NotImplementedError("marking pages for deletions is not yet available.")
+ raise e
+
def loadDeletedRevisions(self):
"""Retrieve all deleted revisions for this Page from Special/Undelete.
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2009-03-01 09:16:30 UTC (rev 6459)
+++ branches/rewrite/pywikibot/site.py 2009-03-01 09:19:24 UTC (rev 6460)
@@ -2420,11 +2420,11 @@
"""
try:
self.login(sysop=True)
- except pywikibot.Error, e:
- raise Error("delete: Unable to login as sysop (%s)"
+ except pywikibot.NoUsername, e:
+ raise NoUsername("delete: Unable to login as sysop (%s)"
% e.__class__.__name__)
if not self.logged_in(sysop=True):
- raise Error("delete: Unable to login as sysop")
+ raise NoUsername("delete: Unable to login as sysop")
token = self.token("delete")
req = api.Request(site=self, action="delete", token=token,
title=page.title(withSection=False),