Revision: 5287
Author: russblau
Date: 2008-04-30 17:15:31 +0000 (Wed, 30 Apr 2008)
Log Message:
-----------
threading wasn't working; take it out to make debugging easier.
Modified Paths:
--------------
branches/rewrite/pywikibot/site.py
branches/rewrite/pywikibot/tools.py
Added Paths:
-----------
branches/rewrite/pywikibot/scripts/
branches/rewrite/pywikibot/scripts/__init__.py
Added: branches/rewrite/pywikibot/scripts/__init__.py
===================================================================
--- branches/rewrite/pywikibot/scripts/__init__.py (rev 0)
+++ branches/rewrite/pywikibot/scripts/__init__.py 2008-04-30 17:15:31 UTC (rev 5287)
@@ -0,0 +1 @@
+# THIS DIRECTORY IS TO HOLD BOT SCRIPTS FOR THE NEW FRAMEWORK
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2008-04-30 08:39:17 UTC (rev 5286)
+++ branches/rewrite/pywikibot/site.py 2008-04-30 17:15:31 UTC (rev 5287)
@@ -578,65 +578,48 @@
api.update_page(target, pagedata)
page._redir = target
- def preloadpages(self, pagelist, size=60, lookahead=0):
+ def preloadpages(self, pagelist, groupsize=60):
"""Return a generator to a list of preloaded pages.
@param pagelist: an iterable that returns Page objects
- @param size: how many Pages to query at a time
- @type size: int
- @param lookahead: if greater than zero, preload pages in a
- separate thread for greater responsiveness; higher values
- result in more aggressive preloading
- @type lookahead: int
+ @param groupsize: how many Pages to query at a time
+ @type groupsize: int
"""
- from pywikibot.tools import itergroup, ThreadedGenerator
- gen = ThreadedGenerator(target=itergroup,
- args=(pagelist, size),
- qsize=lookahead)
- try:
- for sublist in gen:
- pageids = []
- cache = {}
- for p in sublist:
- if pageids is not None:
- if hasattr(p, "_pageid"):
- pageids.append(str(p._pageid))
- else:
- # only use pageids if all pages have them
- pageids = None
- cache[p.title(withSection=False)] = p
- rvgen = api.PropertyGenerator("revisions|info")
- if pageids is not None:
- rvgen.request["pageids"] = "|".join(pageids)
- else:
- rvgen.request["titles"] = "|".join(cache.keys())
- rvgen.request[u"rvprop"] = \
- u"ids|flags|timestamp|user|comment|content"
- for pagedata in rvgen:
- if pagedata['title'] not in cache:
- raise Error(
+ from pywikibot.tools import itergroup
+ for sublist in itergroup(pagelist, groupsize):
+ pageids = [str(p._pageid) for p in sublist if hasattr(p,
"_pageid")]
+ cache = dict((p.title(withSection=False), p) for p in sublist)
+ rvgen = api.PropertyGenerator("revisions|info")
+ if len(pageids) == len(sublist):
+ # only use pageids if all pages have them
+ rvgen.request["pageids"] = "|".join(pageids)
+ else:
+ rvgen.request["titles"] = "|".join(cache.keys())
+ rvgen.request[u"rvprop"] = \
+ u"ids|flags|timestamp|user|comment|content"
+ for pagedata in rvgen:
+ if pagedata['title'] not in cache:
+ raise Error(
u"preloadpages: Query returned unexpected title
'%s'"
- % pagedata['title']
- )
- page = cache[pagedata['title']]
- api.update_page(page, pagedata)
- if 'revisions' in pagedata: # true if page exists
- for rev in pagedata['revisions']:
- revision = pywikibot.page.Revision(
- revid=rev['revid'],
- timestamp=rev['timestamp'],
- user=rev['user'],
- anon=rev.has_key('anon'),
- comment=rev.get('comment',
u''),
- minor=rev.has_key('minor'),
- text=rev.get('*', None)
- )
- page._revisions[revision.revid] = revision
- page._revid = revision.revid
- yield page
- finally:
- gen.stop()
+ % pagedata['title']
+ )
+ page = cache[pagedata['title']]
+ api.update_page(page, pagedata)
+ if 'revisions' in pagedata: # true if page exists
+ for rev in pagedata['revisions']:
+ revision = pywikibot.page.Revision(
+ revid=rev['revid'],
+ timestamp=rev['timestamp'],
+ user=rev['user'],
+ anon=rev.has_key('anon'),
+ comment=rev.get('comment',
u''),
+ minor=rev.has_key('minor'),
+ text=rev.get('*', None)
+ )
+ page._revisions[revision.revid] = revision
+ page._revid = revision.revid
+ yield page
# following group of methods map more-or-less directly to API queries
Modified: branches/rewrite/pywikibot/tools.py
===================================================================
--- branches/rewrite/pywikibot/tools.py 2008-04-30 08:39:17 UTC (rev 5286)
+++ branches/rewrite/pywikibot/tools.py 2008-04-30 17:15:31 UTC (rev 5287)
@@ -114,14 +114,14 @@
StopIteration
"""
- chunk = []
- for item in iter(iterable):
- chunk.append(item)
- if len(chunk) == size:
- yield chunk
- chunk = []
- if chunk:
- yield chunk
+ group = []
+ for item in iterable:
+ group.append(item)
+ if len(group) == size:
+ yield group
+ group = []
+ if group:
+ yield group
if __name__ == "__main__":
Show replies by thread