http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11059
Revision: 11059
Author: xqt
Date: 2013-02-09 15:24:52 +0000 (Sat, 09 Feb 2013)
Log Message:
-----------
some updates from trunk
Modified Paths:
--------------
branches/rewrite/scripts/interwiki.py
Modified: branches/rewrite/scripts/interwiki.py
===================================================================
--- branches/rewrite/scripts/interwiki.py 2013-02-09 15:24:04 UTC (rev 11058)
+++ branches/rewrite/scripts/interwiki.py 2013-02-09 15:24:52 UTC (rev 11059)
@@ -1315,7 +1315,8 @@
iw = page.langlinks()
except pywikibot.NoSuchSite:
if not globalvar.quiet:
- pywikibot.output(u"NOTE: site %s does not exist" % page.site)
+ pywikibot.output(u"NOTE: site %s does not exist."
+ % page.site)
continue
(skip, alternativePage) = self.disambigMismatch(page, counter)
@@ -1988,10 +1989,12 @@
try:
linkedPages = set(pywikibot.Page(l) for l in page.iterlanglinks())
except pywikibot.NoPage:
- pywikibot.output(u"WARNING: Page %s does no longer exist?!" % page)
+ pywikibot.output(
+ u"WARNING: Page %s does no longer exist?!" % page)
break
- # To speed things up, create a dictionary which maps sites to pages.
- # This assumes that there is only one interwiki link per language.
+ # To speed things up, create a dictionary which maps sites
+ # to pages. This assumes that there is only one interwiki
+ # link per language.
linkedPagesDict = {}
for linkedPage in linkedPages:
linkedPagesDict[linkedPage.site] = linkedPage
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11058
Revision: 11058
Author: xqt
Date: 2013-02-09 15:24:04 +0000 (Sat, 09 Feb 2013)
Log Message:
-----------
since we have -async option we don't need to make another get request while finishing a subject; update from rewrite r10660
Modified Paths:
--------------
trunk/pywikipedia/interwiki.py
Modified: trunk/pywikipedia/interwiki.py
===================================================================
--- trunk/pywikipedia/interwiki.py 2013-02-09 13:57:31 UTC (rev 11057)
+++ trunk/pywikipedia/interwiki.py 2013-02-09 15:24:04 UTC (rev 11058)
@@ -1552,14 +1552,11 @@
break
return result
- def finish(self, bot=None):
+ def finish(self):
"""Round up the subject, making any necessary changes. This method
should be called exactly once after the todo list has gone empty.
- This contains a shortcut: if a subject list is given in the argument
- bot, just before submitting a page change to the live wiki it is
- checked whether we will have to wait. If that is the case, the bot will
- be told to make another get request first."""
+ """
#from clean_sandbox
def minutesDiff(time1, time2):
@@ -1620,7 +1617,7 @@
if not self.originPage.site.family.interwiki_forward:
new[self.originPage.site] = self.originPage
- #self.replaceLinks(self.originPage, new, True, bot)
+ #self.replaceLinks(self.originPage, new, True)
updatedSites = []
notUpdatedSites = []
@@ -1639,7 +1636,7 @@
lclSiteDone = True # even if we fail the update
if site.family.name in config.usernames and site.lang in config.usernames[site.family.name]:
try:
- if self.replaceLinks(new[site], new, bot):
+ if self.replaceLinks(new[site], new):
updatedSites.append(site)
if site != lclSite:
frgnSiteDone = True
@@ -1665,7 +1662,7 @@
(globalvar.needlimit and \
len(adding) + len(modifying) >= globalvar.needlimit +1):
try:
- if self.replaceLinks(new[site], new, bot):
+ if self.replaceLinks(new[site], new):
updatedSites.append(site)
except SaveError:
notUpdatedSites.append(site)
@@ -1730,7 +1727,7 @@
not site.has_transcluded_data:
# Try to do the changes
try:
- if self.replaceLinks(page, new, bot):
+ if self.replaceLinks(page, new):
# Page was changed
updatedSites.append(site)
except SaveError:
@@ -1768,7 +1765,7 @@
if hasattr(page, '_contents'):
del page._contents
- def replaceLinks(self, page, newPages, bot):
+ def replaceLinks(self, page, newPages):
"""
Returns True if saving was successful.
"""
@@ -1948,17 +1945,6 @@
answer = 'y'
# If we got permission to submit, do so
if answer == 'y':
- # Check whether we will have to wait for pywikibot. If so, make
- # another get-query first.
- if bot:
- while pywikibot.get_throttle.waittime() + 2.0 < pywikibot.put_throttle.waittime():
- if not globalvar.quiet or pywikibot.verbose:
- pywikibot.output(
- u"NOTE: Performing a recursive query first to save time....")
- qdone = bot.oneQuery()
- if not qdone:
- # Nothing more to do
- break
if not globalvar.quiet or pywikibot.verbose:
pywikibot.output(u"NOTE: Updating live wiki...")
timeout=60
@@ -2284,7 +2270,7 @@
for i in xrange(len(self.subjects)-1, -1, -1):
subj = self.subjects[i]
if subj.isDone():
- subj.finish(self)
+ subj.finish()
subj.clean()
del self.subjects[i]
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11057
Revision: 11057
Author: xqt
Date: 2013-02-09 13:57:31 +0000 (Sat, 09 Feb 2013)
Log Message:
-----------
some steps for data reporsitory
Modified Paths:
--------------
trunk/pywikipedia/interwiki.py
Modified: trunk/pywikipedia/interwiki.py
===================================================================
--- trunk/pywikipedia/interwiki.py 2013-02-09 13:41:50 UTC (rev 11056)
+++ trunk/pywikipedia/interwiki.py 2013-02-09 13:57:31 UTC (rev 11057)
@@ -163,6 +163,8 @@
for the format, one can for example give "en:something" or
"20:" as hint.
+ -repository Include data repository
+
-same looks over all 'serious' languages for the same title.
-same is equivalent to -hint:all:
(note: without ending colon)
@@ -333,7 +335,7 @@
# (C) Rob W.W. Hooft, 2003
# (C) Daniel Herding, 2004
# (C) Yuri Astrakhan, 2005-2006
-# (C) xqt, 2009-2012
+# (C) xqt, 2009-2013
# (C) Pywikipedia bot team, 2007-2013
#
# Distributed under the terms of the MIT license.
@@ -498,6 +500,7 @@
restoreAll = False
async = False
summary = u''
+ repository = False
def readOptions(self, arg):
""" Read all commandline parameters for the global container """
@@ -522,6 +525,8 @@
self.same = True
elif arg == '-wiktionary':
self.same = 'wiktionary'
+ elif arg == '-repository':
+ self.repository = True
elif arg == '-untranslated':
self.untranslated = True
elif arg == '-untranslatedonly':
@@ -831,6 +836,8 @@
self.todo = PageTree()
if originPage:
self.todo.add(originPage)
+ if globalvar.repository:
+ self.todo.add(pywikibot.DataPage(originPage))
# done is a list of all pages that have been analyzed and that
# are known to belong to this subject.
@@ -1202,6 +1209,8 @@
"""
# Loop over all the pages that should have been taken care of
for page in self.pending:
+ if page.title == None: ### seems a DataPage
+ page.get() ### get it's title (and content)
# Mark the page as done
self.done.add(page)
@@ -1322,7 +1331,8 @@
iw = page.interwiki()
except pywikibot.NoSuchSite:
if not globalvar.quiet or pywikibot.verbose:
- pywikibot.output(u"NOTE: site %s does not exist" % page.site)
+ pywikibot.output(u"NOTE: site %s does not exist."
+ % page.site)
continue
(skip, alternativePage) = self.disambigMismatch(page, counter)
@@ -2022,10 +2032,12 @@
try:
linkedPages = set(page.interwiki())
except pywikibot.NoPage:
- pywikibot.output(u"WARNING: Page %s does no longer exist?!" % page)
+ pywikibot.output(
+ u"WARNING: Page %s does no longer exist?!" % page)
break
- # To speed things up, create a dictionary which maps sites to pages.
- # This assumes that there is only one interwiki link per language.
+ # To speed things up, create a dictionary which maps sites
+ # to pages. This assumes that there is only one interwiki
+ # link per language.
linkedPagesDict = {}
for linkedPage in linkedPages:
linkedPagesDict[linkedPage.site] = linkedPage
@@ -2038,10 +2050,11 @@
% (page.site.family.name,
page, expectedPage, linkedPage))
except KeyError:
- pywikibot.output(
- u"WARNING: %s: %s does not link to %s"
- % (page.site.family.name,
- page, expectedPage))
+ if not expectedPage.site.is_data_repository():
+ pywikibot.output(
+ u"WARNING: %s: %s does not link to %s"
+ % (page.site.family.name,
+ page, expectedPage))
# Check for superfluous links
for linkedPage in linkedPages:
if linkedPage not in expectedPages:
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11055
Revision: 11055
Author: xqt
Date: 2013-02-08 14:43:56 +0000 (Fri, 08 Feb 2013)
Log Message:
-----------
use pywikibot.calledModuleName for the script
Modified Paths:
--------------
branches/rewrite/pywikibot/comms/http.py
Modified: branches/rewrite/pywikibot/comms/http.py
===================================================================
--- branches/rewrite/pywikibot/comms/http.py 2013-02-08 14:31:38 UTC (rev 11054)
+++ branches/rewrite/pywikibot/comms/http.py 2013-02-08 14:43:56 UTC (rev 11055)
@@ -40,14 +40,11 @@
# global variables
# the User-agent: header. The default is
-# '<script>/<revision> Pywikipediabot/2.0', where '<script>' is the tail
-# path component and file name of the currently executing script and
-# revision is the SVN revision of Pywikipediabot.
+# '<script>/<revision> Pywikipediabot/2.0', where '<script>' is the currently
+# executing script and version is the SVN revision of Pywikipediabot.
USER_AGENT_FORMAT = '{script}/r{version[rev]} Pywikipediabot/2.0'
-useragent = (USER_AGENT_FORMAT.format(
- script=('-'.join(pywikibot.version.get_executing_script())),
- version=pywikibot.version.getversiondict()
-))
+useragent = USER_AGENT_FORMAT.format(script=pywikibot.calledModuleName(),
+ version=pywikibot.version.getversiondict())
numthreads = 1
threads = []