August 2009 - Pywikipedia-svn - lists.wikimedia.org

SVN: [7194] trunk/pywikipedia/families/wikipedia_family.py
by purodha＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7194 Author: purodha Date: 2009-08-31 21:52:10 +0000 (Mon, 31 Aug 2009) Log Message: ----------- Applying patch #2809532 of nakor-wikipedia fixing the interwiki sort order in the fy.wikipedia Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-31 20:49:50 UTC (rev 7193) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-31 21:52:10 UTC (rev 7194) @@ -928,6 +928,10 @@ # Order for fy: alphabetical by code, but y counts as i def fycomp(x,y): + if x == 'nb': + x = 'no' + if y == 'nb': + y = 'no' x = x.replace("y","i")+x.count("y")*"!" y = y.replace("y","i")+y.count("y")*"!" return cmp(x,y)

1 0

SVN: [7193] trunk/pywikipedia/pagegenerators.py
by multichill＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7193 Author: multichill Date: 2009-08-31 20:49:50 +0000 (Mon, 31 Aug 2009) Log Message: ----------- Restore previous behavior: Work from new to old. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2009-08-31 20:00:38 UTC (rev 7192) +++ trunk/pywikipedia/pagegenerators.py 2009-08-31 20:49:50 UTC (rev 7193) @@ -494,7 +494,7 @@ 'ucuser': username, 'ucprop': 'title', 'uclimit': int(number), - 'ucdir': 'newer', + 'ucdir': 'older', }

1 0

SVN: [7192] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7192 Author: alexsh Date: 2009-08-31 20:00:38 +0000 (Mon, 31 Aug 2009) Log Message: ----------- Page().getVersionHistory(): * remove duplicate codes * use a temporary list to save revisions and dump into self._versionhistory or self._versionhistoryearliest after the process. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-31 15:40:11 UTC (rev 7191) +++ trunk/pywikipedia/wikipedia.py 2009-08-31 20:00:38 UTC (rev 7192) @@ -2283,7 +2283,6 @@ unless getAll is True. """ - site = self.site() # regular expression matching one edit in the version history. # results will have 4 groups: oldid, edit date/time, user name, and edit @@ -2297,6 +2296,7 @@ startFromPage = None thisHistoryDone = False skip = False # Used in determining whether we need to skip the first page + dataQuery = [] RLinkToNextPage = re.compile('&offset=(.*?)&') @@ -2309,6 +2309,7 @@ # Cause a reload, or at least make the loop run thisHistoryDone = False skip = True + dataQuery = self._versionhistoryearliest else: thisHistoryDone = True elif not hasattr(self, '_versionhistory') or forceReload: @@ -2317,11 +2318,12 @@ # Cause a reload, or at least make the loop run thisHistoryDone = False skip = True + dataQuery = self._versionhistory else: thisHistoryDone = True while not thisHistoryDone: - path = site.family.version_history_address(self.site().language(), self.urlname(), revCount) + path = self.site().family.version_history_address(self.site().language(), self.urlname(), revCount) if reverseOrder: path += '&dir=prev' @@ -2341,108 +2343,60 @@ else: output(u'Getting version history of %s' % self.aslink(forceInterwiki = True)) - txt = site.getUrl(path) + txt = self.site().getUrl(path) # save a copy of the text self_txt = txt - if reverseOrder: - # If we are getting all of the page history... - if getAll: - if len(self._versionhistoryearliest) == 0: - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True - - edits = editR.findall(self_txt) - edits.reverse() - for edit in edits: - self._versionhistoryearliest.append(edit) - if len(edits) < revCount: - thisHistoryDone = True - else: - if not skip: - edits = editR.findall(self_txt) - edits.reverse() - for edit in edits: - self._versionhistoryearliest.append(edit) - if len(edits) < revCount: - thisHistoryDone = True - - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True - - else: - # Skip the first page only, - skip = False - - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True + # If we are getting all of the page history... + if getAll: + #Find the nextPage link, if not exist, the page is last history page + matchObj = RLinkToNextPage.search(self_txt) + if matchObj: + startFromPage = matchObj.group(1) else: - # If we are not getting all, we stop on the first page. - for edit in editR.findall(self_txt): - self._versionhistoryearliest.append(edit) - self._versionhistoryearliest.reverse() - thisHistoryDone = True - else: - # If we are getting all of the page history... - if getAll: - if len(self._versionhistory) == 0: - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True + if len(dataQuery) == 0: + edits = editR.findall(self_txt) + if reverseOrder: + edits.reverse() + #for edit in edits: + dataQuery.extend([edit for edit in edits]) + if len(edits) < revCount: + thisHistoryDone = True + else: + if not skip: edits = editR.findall(self_txt) - for edit in edits: - self._versionhistory.append(edit) + if reverseOrder: + edits.reverse() + #for edit in edits: + dataQuery.extend([edit for edit in edits]) if len(edits) < revCount: thisHistoryDone = True else: - if not skip: - edits = editR.findall(self_txt) - for edit in edits: - self._versionhistory.append(edit) - if len(edits) < revCount: - thisHistoryDone = True + # Skip the first page only, + skip = False + else: + # If we are not getting all, we stop on the first page. + #for edit in editR.findall(self_txt): + dataQuery.extend([edit for edit in editR.findall(self_txt)] ) + if reverseOrder: + dataQuery.reverse() + thisHistoryDone = True - matchObj = RLinkToNextPage.findall(self_txt) - if len(matchObj) >= 2: - startFromPage = matchObj[1] - else: - thisHistoryDone = True - else: - # Skip the first page only, - skip = False - - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True - else: - # If we are not getting all, we stop on the first page. - for edit in editR.findall(self_txt): - self._versionhistory.append(edit) - - thisHistoryDone = True - if reverseOrder: # Return only revCount edits, even if the version history is extensive + if dataQuery != []: + self._versionhistoryearliest = dataQuery + del dataQuery if len(self._versionhistoryearliest) > revCount and not getAll: return self._versionhistoryearliest[0:revCount] return self._versionhistoryearliest + if dataQuery != []: + self._versionhistory = dataQuery + del dataQuery # Return only revCount edits, even if the version history is extensive if len(self._versionhistory) > revCount and not getAll: return self._versionhistory[0:revCount]

1 0

SVN: [7191] trunk/pywikipedia/flickrripper.py
by multichill＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7191 Author: multichill Date: 2009-08-31 15:40:11 +0000 (Mon, 31 Aug 2009) Log Message: ----------- More comments and prevent naming collisions Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-31 15:09:31 UTC (rev 7190) +++ trunk/pywikipedia/flickrripper.py 2009-08-31 15:40:11 UTC (rev 7191) @@ -42,6 +42,9 @@ def getPhoto(flickr = None, photo_id = ''): ''' Get the photo info and the photo sizes so we can use these later on + + TODO: Add exception handling + ''' photoInfo = flickr.photos_getInfo(photo_id=photo_id) #xml.etree.ElementTree.dump(photoInfo) @@ -52,6 +55,8 @@ def isAllowedLicense(photoInfo = None): ''' Check if the image contains the right license + + TODO: Maybe add more licenses ''' license = photoInfo.find('photo').attrib['license'] if license == '4' or license == '5': @@ -72,10 +77,20 @@ return url def downloadPhoto(photoUrl=''): + ''' + Download the photo and store it in a StrinIO.StringIO object. + + TODO: Add exception handling + ''' imageFile=urllib.urlopen(photoUrl).read() return StringIO.StringIO(imageFile) def findDuplicateImages(photo=None, site=wikipedia.getSite()): + ''' + Takes the photo, calculates the SHA1 hash and asks the mediawiki api for a list of duplicates. + + TODO: Add exception handling, fix site thing + ''' result = [] hashObject = hashlib.sha1() hashObject.update(photo.getvalue()) @@ -87,7 +102,7 @@ 'aisha1' : sha1Hash, 'aiprop' : '', } - data = query.GetData(params, wikipedia.getSite(), useAPI = True, encodeTitle = False) + data = query.GetData(params, site=wikipedia.getSite(), useAPI = True, encodeTitle = False) for image in data['query']['allimages']: result.append(image['name']) return result @@ -105,6 +120,8 @@ def getFlinfoDescription(photo_id = 0): ''' Get the description from http://wikipedia.ramselehof.de/flinfo.php + + TODO: Add exception handling, try a couple of times ''' parameters = urllib.urlencode({'id' : photo_id, 'raw' : 'on'}) @@ -113,9 +130,11 @@ #print rawDescription.decode('utf-8') return rawDescription.decode('utf-8') -def getFilename(photoInfo=None): +def getFilename(photoInfo=None, site=wikipedia.getSite()): ''' - Build a good filename for the upload based on the username and the title + Build a good filename for the upload based on the username and the title. + Prevents naming collisions. + ''' username = photoInfo.find('photo').find('owner').attrib['username'] title = photoInfo.find('photo').find('title').text @@ -124,9 +143,20 @@ else: title = u'' - return u'Flickr - %s - %s.jpg' % (username, title) + if (wikipedia.Page(title=u'File:Flickr - %s - %s.jpg' % (username, title), site=wikipedia.getSite()).exists()): + i = 1 + while True: + if (wikipedia.Page(title=u'File:Flickr - %s - %s (%s).jpg' % (username, title, str(i)), site=wikipedia.getSite()).exists()): + i = i + 1 + else: + return u'Flickr - %s - %s (%s).jpg' % (username, title, str(i)) + else: + return u'Flickr - %s - %s.jpg' % (username, title) def cleanUpTitle(title): + ''' + Clean up the title of a potential mediawiki page. Otherwise the title of the page might not be allowed by the software. + ''' title = title.strip() title = re.sub("[<{\\[]", "(", title) @@ -168,6 +198,9 @@ return description def processPhoto(flickr=None, photo_id=u'', flickrreview=False, reviewer=u'', override=u''): + ''' + Process a single Flickr photo + ''' if(photo_id): print photo_id (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) @@ -202,6 +235,9 @@ return 0 class Tkdialog: + ''' + The user dialog. + ''' def __init__(self, photoDescription, photo, filename): self.root=Tk() #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) @@ -257,6 +293,9 @@ self.descriptionScrollbar.grid(row=14, column=5) def getImage(self, photo, width, height): + ''' + Take the StringIO object and build an imageTK thumbnail + ''' image = Image.open(photo) image.thumbnail((width, height)) imageTk = ImageTk.PhotoImage(image) @@ -285,8 +324,12 @@ return (self.photoDescription, self.filename, self.skip) def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): + ''' + Loop over a set of Flickr photos. + ''' result = [] # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html + # Get the photos in a group if(group_id): #First get the total number of photo's in the group photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') @@ -297,6 +340,7 @@ yield photo.attrib['id'] # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html + # Get the photos in a photoset elif(photoset_id): photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') pages = photos.find('photos').attrib['pages'] @@ -306,6 +350,7 @@ yield photo.attrib['id'] # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html + # Get the (public) photos uploaded by a user elif(user_id): photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') pages = photos.find('photos').attrib['pages'] @@ -316,6 +361,11 @@ return def usage(): + ''' + Print usage information + + TODO : Need more. + ''' wikipedia.output(u"Flickrripper is a tool to transfer flickr photos to Wikimedia Commons") wikipedia.output(u"-group_id:<group_id>\n") wikipedia.output(u"-photoset_id:<photoset_id>\n")

1 0

SVN: [7190] trunk/pywikipedia/flickrripper.py
by multichill＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7190 Author: multichill Date: 2009-08-31 15:09:31 +0000 (Mon, 31 Aug 2009) Log Message: ----------- *Skipping dupes now *Rearranged some code Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-31 11:00:07 UTC (rev 7189) +++ trunk/pywikipedia/flickrripper.py 2009-08-31 15:09:31 UTC (rev 7190) @@ -31,7 +31,7 @@ # __version__ = '$Id$' -import sys, urllib, re, StringIO +import sys, urllib, re, StringIO, hashlib, base64 import wikipedia, config, query, imagerecat, upload import flickrapi # see: http://stuvel.eu/projects/flickrapi @@ -61,6 +61,37 @@ #We don't accept other licenses return False +def getPhotoUrl(photoSizes=None): + ''' + Get the url of the jpg file with the highest resolution + ''' + url = '' + # The assumption is that the largest image is last + for size in photoSizes.find('sizes').findall('size'): + url = size.attrib['source'] + return url + +def downloadPhoto(photoUrl=''): + imageFile=urllib.urlopen(photoUrl).read() + return StringIO.StringIO(imageFile) + +def findDuplicateImages(photo=None, site=wikipedia.getSite()): + result = [] + hashObject = hashlib.sha1() + hashObject.update(photo.getvalue()) + sha1Hash = base64.b16encode(hashObject.digest()) + + params = { + 'action' : 'query', + 'list' : 'allimages', + 'aisha1' : sha1Hash, + 'aiprop' : '', + } + data = query.GetData(params, wikipedia.getSite(), useAPI = True, encodeTitle = False) + for image in data['query']['allimages']: + result.append(image['name']) + return result + def getTags(photoInfo = None): ''' Get all the tags on a photo @@ -116,17 +147,8 @@ title = title.replace(" ", "_") return title + -def getPhotoUrl(photoSizes=None): - ''' - Get the url of the jpg file with the highest resolution - ''' - url = '' - # The assumption is that the largest image is last - for size in photoSizes.find('sizes').findall('size'): - url = size.attrib['source'] - return url - def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): ''' Build the final description for the image. The description is based on the info from flickrinfo and improved. @@ -143,57 +165,28 @@ if(reviewer): description = description.replace(u'{{flickrreview}}', u'{{flickrreview|' + reviewer + '|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}') description = description.replace(u'\r\n', u'\n') - return description + return description -def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): - result = [] - # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html - if(group_id): - #First get the total number of photo's in the group - photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - - # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html - elif(photoset_id): - photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - - # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html - elif(user_id): - photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - return - - def processPhoto(flickr=None, photo_id=u'', flickrreview=False, reviewer=u'', override=u''): if(photo_id): print photo_id (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) if (isAllowedLicense(photoInfo=photoInfo) or override): - # Tags not needed atm - #tags=getTags(photoInfo=photoInfo) + #Get the url of the largest photo + photoUrl = getPhotoUrl(photoSizes=photoSizes) + #Should download the photo only once + photo = downloadPhoto(photoUrl=photoUrl) - flinfoDescription = getFlinfoDescription(photo_id=photo_id) - - filename = getFilename(photoInfo=photoInfo) - #print filename - photoUrl = getPhotoUrl(photoSizes=photoSizes) - #print photoUrl - photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) - #wikipedia.output(photoDescription) - (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() + #Don't upload duplicate images, should add override option + duplicates = findDuplicateImages(photo=photo) + if duplicates: + wikipedia.output(u'Found duplicate image at %s' % duplicates.pop()) + else: + filename = getFilename(photoInfo=photoInfo) + flinfoDescription = getFlinfoDescription(photo_id=photo_id) + photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) + #wikipedia.output(photoDescription) + (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photo, filename).run() #wikipedia.output(newPhotoDescription) #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): # I should probably check if the hash is the same and if not upload it under a different name @@ -202,13 +195,14 @@ #Do the actual upload #Would be nice to check before I upload if the file is already at Commons #Not that important for this program, but maybe for derived programs - if not skip: - bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) - bot.upload_image(debug=False) - return 0 + if not skip: + bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) + bot.upload_image(debug=False) + return 1 + return 0 class Tkdialog: - def __init__(self, photoDescription, photoUrl, filename): + def __init__(self, photoDescription, photo, filename): self.root=Tk() #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) self.root.geometry("%ix%i+10-10"%(config.tkhorsize, config.tkvertsize)) @@ -216,13 +210,13 @@ self.root.title(filename) self.photoDescription = photoDescription self.filename = filename - self.photoUrl = photoUrl + self.photo = photo self.skip=False self.exit=False ## Init of the widgets # The image - self.image=self.getImage(self.photoUrl, 800, 600) + self.image=self.getImage(self.photo, 800, 600) self.imagePanel=Label(self.root, image=self.image) self.imagePanel.image = self.image @@ -262,12 +256,10 @@ self.descriptionField.grid(row=14, column=1, columnspan=3) self.descriptionScrollbar.grid(row=14, column=5) - def getImage(self, url, width, height): - image=urllib.urlopen(url).read() - output = StringIO.StringIO(image) - image2 = Image.open(output) - image2.thumbnail((width, height)) - imageTk = ImageTk.PhotoImage(image2) + def getImage(self, photo, width, height): + image = Image.open(photo) + image.thumbnail((width, height)) + imageTk = ImageTk.PhotoImage(image) return imageTk def okFile(self): @@ -292,7 +284,37 @@ self.root.mainloop() return (self.photoDescription, self.filename, self.skip) +def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): + result = [] + # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html + if(group_id): + #First get the total number of photo's in the group + photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + for i in range(1, int(pages)): + for photo in flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html + elif(photoset_id): + photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html + elif(user_id): + photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + return + def usage(): wikipedia.output(u"Flickrripper is a tool to transfer flickr photos to Wikimedia Commons") wikipedia.output(u"-group_id:<group_id>\n")

1 0

SVN: [7189] trunk/pywikipedia/imageuncat.py
by multichill＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7189 Author: multichill Date: 2009-08-31 11:00:07 +0000 (Mon, 31 Aug 2009) Log Message: ----------- Raised the delay to 2 hours. Modified Paths: -------------- trunk/pywikipedia/imageuncat.py Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2009-08-31 09:30:38 UTC (rev 7188) +++ trunk/pywikipedia/imageuncat.py 2009-08-31 11:00:07 UTC (rev 7189) @@ -1254,7 +1254,7 @@ return pagegenerators.PagesFromTitlesGenerator(result, site) -def recentChanges(site = None, delay=60, block=70): +def recentChanges(site = None, delay=0, block=70): ''' Return a pagegenerator containing all the images edited in a certain timespan. The delay is the amount of minutes to wait and the block is the timespan to return images in. @@ -1348,7 +1348,7 @@ if arg.startswith('-yesterday'): generator = uploadedYesterday(site) elif arg.startswith('-recentchanges'): - generator = recentChanges(site) + generator = recentChanges(site=site, delay=120) else: genFactory.handleArg(arg)

1 0

SVN: [7188] trunk/pywikipedia/flickrripper.py
by multichill＠svn.wikimedia.org 31 Aug '09

31 Aug '09

Revision: 7188 Author: multichill Date: 2009-08-31 09:30:38 +0000 (Mon, 31 Aug 2009) Log Message: ----------- svn properties Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Property Changed: ---------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-28 23:11:46 UTC (rev 7187) +++ trunk/pywikipedia/flickrripper.py 2009-08-31 09:30:38 UTC (rev 7188) @@ -1,392 +1,392 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -''' -Tool to copy a flickr stream to Commons - -# Get a set to work on (start with just a username). -# * Make it possible to delimit the set (from/to) -#For each image -#*Check the license -#*Check if it isn't already on Commons -#*Build suggested filename -#**Check for name collision and maybe alter it -#*Pull description from Flinfo -#*Show image and description to user -#**Add a nice hotcat lookalike for the adding of categories -#**Filter the categories -#*Upload the image - -Todo: -*Check if the image is already uploaded (SHA hash) -*Check and prevent filename collisions -**Initial suggestion -**User input -*Filter the categories - -''' -# -# (C) Multichill, 2009 -# -# Distributed under the terms of the MIT license. -# -__version__ = '$Id: upload_to_commons.py 69 2009-08-23 11:44:26Z multichill $' - -import sys, urllib, re, StringIO -import wikipedia, config, query, imagerecat, upload - -import flickrapi # see: http://stuvel.eu/projects/flickrapi -import xml.etree.ElementTree -from Tkinter import * -from PIL import Image, ImageTk # see: http://www.pythonware.com/products/pil/ - -def getPhoto(flickr = None, photo_id = ''): - ''' - Get the photo info and the photo sizes so we can use these later on - ''' - photoInfo = flickr.photos_getInfo(photo_id=photo_id) - #xml.etree.ElementTree.dump(photoInfo) - photoSizes = flickr.photos_getSizes(photo_id=photo_id) - #xml.etree.ElementTree.dump(photoSizes) - return (photoInfo, photoSizes) - -def isAllowedLicense(photoInfo = None): - ''' - Check if the image contains the right license - ''' - license = photoInfo.find('photo').attrib['license'] - if license == '4' or license == '5': - #Is cc-by or cc-by-sa - return True - else: - #We don't accept other licenses - return False - -def getTags(photoInfo = None): - ''' - Get all the tags on a photo - ''' - result = [] - for tag in photoInfo.find('photo').find('tags').findall('tag'): - result.append(tag.text.lower()) - - return result - -def getFlinfoDescription(photo_id = 0): - ''' - Get the description from http://wikipedia.ramselehof.de/flinfo.php - ''' - parameters = urllib.urlencode({'id' : photo_id, 'raw' : 'on'}) - - #print 'Flinfo gaat nu aan de slag' - rawDescription = urllib.urlopen("http://wikipedia.ramselehof.de/flinfo.php?%s" % parameters).read() - #print rawDescription.decode('utf-8') - return rawDescription.decode('utf-8') - -def getFilename(photoInfo=None): - ''' - Build a good filename for the upload based on the username and the title - ''' - username = photoInfo.find('photo').find('owner').attrib['username'] - title = photoInfo.find('photo').find('title').text - if title: - title = cleanUpTitle(title) - else: - title = u'' - - return u'Flickr - %s - %s.jpg' % (username, title) - -def cleanUpTitle(title): - title = title.strip() - - title = re.sub("[<{\\[]", "(", title) - title = re.sub("[>}\\]]", ")", title) - title = re.sub("[ _]?\$!\$", "", title) - title = re.sub(",:[ _]", ", ", title) - title = re.sub("[;:][ _]", ", ", title) - title = re.sub("[\t\n ]+", " ", title) - title = re.sub("[\r\n ]+", " ", title) - title = re.sub("[\n]+", "", title) - title = re.sub("[?!]([.\"]|$)", "\\1", title) - title = re.sub("[&#%?!]", "^", title) - title = re.sub("[;]", ",", title) - title = re.sub("[/+\\\\:]", "-", title) - title = re.sub("--+", "-", title) - title = re.sub(",,+", ",", title) - title = re.sub("[-,^]([.]|$)", "\\1", title) - title = title.replace(" ", "_") - - return title - -def getPhotoUrl(photoSizes=None): - ''' - Get the url of the jpg file with the highest resolution - ''' - url = '' - # The assumption is that the largest image is last - for size in photoSizes.find('sizes').findall('size'): - url = size.attrib['source'] - return url - -def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): - ''' - Build the final description for the image. The description is based on the info from flickrinfo and improved. - ''' - description = flinfoDescription - - if(override): - description = description.replace(u'{{cc-by-sa-2.0}}\n', u'') - description = description.replace(u'{{cc-by-2.0}}\n', u'') - description = description.replace(u'{{flickrreview}}\n', u'') - description = description.replace(u'{{copyvio|Flickr, licensed as "All Rights Reserved" which is not a free license --~~~~}}\n', u'') - description = description.replace(u'=={{int:license}}==', u'=={{int:license}}==\n' + override) - elif(flickrreview): - if(reviewer): - description = description.replace(u'{{flickrreview}}', u'{{flickrreview|' + reviewer + '|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}') - description = description.replace(u'\r\n', u'\n') - return description - -def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): - result = [] - # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html - if(group_id): - #First get the total number of photo's in the group - photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - - # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html - elif(photoset_id): - photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - - # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html - elif(user_id): - photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') - pages = photos.find('photos').attrib['pages'] - - for i in range(1, int(pages)): - for photo in flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page=i).find('photos').getchildren(): - yield photo.attrib['id'] - return - - -def processPhoto(flickr=None, photo_id=u'', flickrreview=False, reviewer=u'', override=u''): - if(photo_id): - print photo_id - (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) - if (isAllowedLicense(photoInfo=photoInfo) or override): - # Tags not needed atm - #tags=getTags(photoInfo=photoInfo) - - flinfoDescription = getFlinfoDescription(photo_id=photo_id) - - filename = getFilename(photoInfo=photoInfo) - #print filename - photoUrl = getPhotoUrl(photoSizes=photoSizes) - #print photoUrl - photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) - #wikipedia.output(photoDescription) - (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() - #wikipedia.output(newPhotoDescription) - #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): - # I should probably check if the hash is the same and if not upload it under a different name - #wikipedia.output(u'File:' + filename + u' already exists!') - #else: - #Do the actual upload - #Would be nice to check before I upload if the file is already at Commons - #Not that important for this program, but maybe for derived programs - if not skip: - bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) - bot.upload_image(debug=False) - return 0 - -class Tkdialog: - def __init__(self, photoDescription, photoUrl, filename): - self.root=Tk() - #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) - self.root.geometry("%ix%i+10-10"%(config.tkhorsize, config.tkvertsize)) - - self.root.title(filename) - self.photoDescription = photoDescription - self.filename = filename - self.photoUrl = photoUrl - self.skip=False - self.exit=False - - ## Init of the widgets - # The image - self.image=self.getImage(self.photoUrl, 800, 600) - self.imagePanel=Label(self.root, image=self.image) - - self.imagePanel.image = self.image - - # The filename - self.filenameLabel=Label(self.root,text=u"Suggested filename") - self.filenameField=Entry(self.root, width=100) - self.filenameField.insert(END, filename) - - # The description - self.descriptionLabel=Label(self.root,text=u"Suggested description") - self.descriptionScrollbar=Scrollbar(self.root, orient=VERTICAL) - self.descriptionField=Text(self.root) - self.descriptionField.insert(END, photoDescription) - self.descriptionField.config(state=NORMAL, height=12, width=100, padx=0, pady=0, wrap=WORD, yscrollcommand=self.descriptionScrollbar.set) - self.descriptionScrollbar.config(command=self.descriptionField.yview) - - # The buttons - self.okButton=Button(self.root, text="OK", command=self.okFile) - self.skipButton=Button(self.root, text="Skip", command=self.skipFile) - - ## Start grid - - # The image - self.imagePanel.grid(row=0, column=0, rowspan=11, columnspan=4) - - # The buttons - self.okButton.grid(row=11, column=1, rowspan=2) - self.skipButton.grid(row=11, column=2, rowspan=2) - - # The filename - self.filenameLabel.grid(row=13, column=0) - self.filenameField.grid(row=13, column=1, columnspan=3) - - # The description - self.descriptionLabel.grid(row=14, column=0) - self.descriptionField.grid(row=14, column=1, columnspan=3) - self.descriptionScrollbar.grid(row=14, column=5) - - def getImage(self, url, width, height): - image=urllib.urlopen(url).read() - output = StringIO.StringIO(image) - image2 = Image.open(output) - image2.thumbnail((width, height)) - imageTk = ImageTk.PhotoImage(image2) - return imageTk - - def okFile(self): - ''' - The user pressed the OK button. - ''' - self.filename=self.filenameField.get() - self.photoDescription=self.descriptionField.get(0.0, END) - self.root.destroy() - - def skipFile(self): - ''' - The user pressed the Skip button. - ''' - self.skip=True - self.root.destroy() - - def run(self): - ''' - Activate the dialog and return the new name and if the image is skipped. - ''' - self.root.mainloop() - return (self.photoDescription, self.filename, self.skip) - - -def usage(): - wikipedia.output(u"Flickrripper is a tool to transfer flickr photos to Wikimedia Commons") - wikipedia.output(u"-group_id:<group_id>\n") - wikipedia.output(u"-photoset_id:<photoset_id>\n") - wikipedia.output(u"-user_id:<user_id>\n") - wikipedia.output(u"-tags:<tag>\n") - return - -def main(): - site = wikipedia.getSite(u'commons', u'commons') - wikipedia.setSite(site) - #imagerecat.initLists() - - #Get the api key - if(config.flickr['api_key']): - flickr = flickrapi.FlickrAPI(config.flickr['api_key']) - else: - wikipedia.output('Flickr api key not found! Get yourself an api key') - wikipedia.output('Any flickr user can get a key at http://www.flickr.com/services/api/keys/apply/') - return - - group_id = u'' - photoset_id = u'' - user_id = u'' - tags = u'' - totalPhotos = 0 - uploadedPhotos = 0 - - # Do we mark the images as reviewed right away? - if config.flickr['review']: - flickrreview = config.flickr['review'] - else: - flickrreview = False - - # Set the Flickr reviewer - if config.flickr['reviewer']: - reviewer = config.flickr['reviewer'] - elif 'commons' in config.sysopnames['commons']: - print config.sysopnames['commons'] - reviewer = config.sysopnames['commons']['commons'] - elif 'commons' in config.usernames['commons']: - reviewer = config.usernames['commons']['commons'] - else: - reviewer = u'' - - override = u'' - - for arg in wikipedia.handleArgs(): - if arg.startswith('-group_id'): - if len(arg) == 9: - group_id = wikipedia.input(u'What is the group_id of the pool?') - else: - group_id = arg[10:] - elif arg.startswith('-photoset_id'): - if len(arg) == 12: - photoset_id = wikipedia.input(u'What is the photoset_id)?') - else: - photoset_id = arg[13:] - elif arg.startswith('-user_id'): - if len(arg) == 8: - user_id = wikipedia.input(u'What is the user_id of the flickr user?') - else: - user_id = arg[9:] - elif arg.startswith('-tags'): - if len(arg) == 5: - tags = wikipedia.input(u'What is the tag you want to filter out (currently only one supported)?') - else: - tags = arg[6:] - elif arg == '-flickrreview': - flickrreview = True - elif arg.startswith('-reviewer'): - if len(arg) == 9: - reviewer = wikipedia.input(u'Who is the reviewer?') - else: - reviewer = arg[10:] - elif arg.startswith('-override'): - if len(arg) == 9: - override = wikipedia.input(u'What is the override text?') - else: - override = arg[10:] - - if user_id or group_id or photoset_id: - for photo_id in getPhotos(flickr=flickr, user_id=user_id, group_id=group_id, photoset_id=photoset_id, tags=tags): - uploadedPhotos = uploadedPhotos + processPhoto(flickr=flickr, photo_id=photo_id, flickrreview=flickrreview, reviewer=reviewer, override=override) - totalPhotos = totalPhotos + 1 - else: - usage() - - wikipedia.output(u'Finished running') - wikipedia.output(u'Total photos: ' + str(totalPhotos)) - wikipedia.output(u'Uploaded photos: ' + str(uploadedPhotos)) - -if __name__ == "__main__": - try: - main() - finally: - wikipedia.stopme() +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Tool to copy a flickr stream to Commons + +# Get a set to work on (start with just a username). +# * Make it possible to delimit the set (from/to) +#For each image +#*Check the license +#*Check if it isn't already on Commons +#*Build suggested filename +#**Check for name collision and maybe alter it +#*Pull description from Flinfo +#*Show image and description to user +#**Add a nice hotcat lookalike for the adding of categories +#**Filter the categories +#*Upload the image + +Todo: +*Check if the image is already uploaded (SHA hash) +*Check and prevent filename collisions +**Initial suggestion +**User input +*Filter the categories + +''' +# +# (C) Multichill, 2009 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import sys, urllib, re, StringIO +import wikipedia, config, query, imagerecat, upload + +import flickrapi # see: http://stuvel.eu/projects/flickrapi +import xml.etree.ElementTree +from Tkinter import * +from PIL import Image, ImageTk # see: http://www.pythonware.com/products/pil/ + +def getPhoto(flickr = None, photo_id = ''): + ''' + Get the photo info and the photo sizes so we can use these later on + ''' + photoInfo = flickr.photos_getInfo(photo_id=photo_id) + #xml.etree.ElementTree.dump(photoInfo) + photoSizes = flickr.photos_getSizes(photo_id=photo_id) + #xml.etree.ElementTree.dump(photoSizes) + return (photoInfo, photoSizes) + +def isAllowedLicense(photoInfo = None): + ''' + Check if the image contains the right license + ''' + license = photoInfo.find('photo').attrib['license'] + if license == '4' or license == '5': + #Is cc-by or cc-by-sa + return True + else: + #We don't accept other licenses + return False + +def getTags(photoInfo = None): + ''' + Get all the tags on a photo + ''' + result = [] + for tag in photoInfo.find('photo').find('tags').findall('tag'): + result.append(tag.text.lower()) + + return result + +def getFlinfoDescription(photo_id = 0): + ''' + Get the description from http://wikipedia.ramselehof.de/flinfo.php + ''' + parameters = urllib.urlencode({'id' : photo_id, 'raw' : 'on'}) + + #print 'Flinfo gaat nu aan de slag' + rawDescription = urllib.urlopen("http://wikipedia.ramselehof.de/flinfo.php?%s" % parameters).read() + #print rawDescription.decode('utf-8') + return rawDescription.decode('utf-8') + +def getFilename(photoInfo=None): + ''' + Build a good filename for the upload based on the username and the title + ''' + username = photoInfo.find('photo').find('owner').attrib['username'] + title = photoInfo.find('photo').find('title').text + if title: + title = cleanUpTitle(title) + else: + title = u'' + + return u'Flickr - %s - %s.jpg' % (username, title) + +def cleanUpTitle(title): + title = title.strip() + + title = re.sub("[<{\\[]", "(", title) + title = re.sub("[>}\\]]", ")", title) + title = re.sub("[ _]?\$!\$", "", title) + title = re.sub(",:[ _]", ", ", title) + title = re.sub("[;:][ _]", ", ", title) + title = re.sub("[\t\n ]+", " ", title) + title = re.sub("[\r\n ]+", " ", title) + title = re.sub("[\n]+", "", title) + title = re.sub("[?!]([.\"]|$)", "\\1", title) + title = re.sub("[&#%?!]", "^", title) + title = re.sub("[;]", ",", title) + title = re.sub("[/+\\\\:]", "-", title) + title = re.sub("--+", "-", title) + title = re.sub(",,+", ",", title) + title = re.sub("[-,^]([.]|$)", "\\1", title) + title = title.replace(" ", "_") + + return title + +def getPhotoUrl(photoSizes=None): + ''' + Get the url of the jpg file with the highest resolution + ''' + url = '' + # The assumption is that the largest image is last + for size in photoSizes.find('sizes').findall('size'): + url = size.attrib['source'] + return url + +def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): + ''' + Build the final description for the image. The description is based on the info from flickrinfo and improved. + ''' + description = flinfoDescription + + if(override): + description = description.replace(u'{{cc-by-sa-2.0}}\n', u'') + description = description.replace(u'{{cc-by-2.0}}\n', u'') + description = description.replace(u'{{flickrreview}}\n', u'') + description = description.replace(u'{{copyvio|Flickr, licensed as "All Rights Reserved" which is not a free license --~~~~}}\n', u'') + description = description.replace(u'=={{int:license}}==', u'=={{int:license}}==\n' + override) + elif(flickrreview): + if(reviewer): + description = description.replace(u'{{flickrreview}}', u'{{flickrreview|' + reviewer + '|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}') + description = description.replace(u'\r\n', u'\n') + return description + +def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): + result = [] + # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html + if(group_id): + #First get the total number of photo's in the group + photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html + elif(photoset_id): + photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html + elif(user_id): + photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + return + + +def processPhoto(flickr=None, photo_id=u'', flickrreview=False, reviewer=u'', override=u''): + if(photo_id): + print photo_id + (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) + if (isAllowedLicense(photoInfo=photoInfo) or override): + # Tags not needed atm + #tags=getTags(photoInfo=photoInfo) + + flinfoDescription = getFlinfoDescription(photo_id=photo_id) + + filename = getFilename(photoInfo=photoInfo) + #print filename + photoUrl = getPhotoUrl(photoSizes=photoSizes) + #print photoUrl + photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) + #wikipedia.output(photoDescription) + (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() + #wikipedia.output(newPhotoDescription) + #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): + # I should probably check if the hash is the same and if not upload it under a different name + #wikipedia.output(u'File:' + filename + u' already exists!') + #else: + #Do the actual upload + #Would be nice to check before I upload if the file is already at Commons + #Not that important for this program, but maybe for derived programs + if not skip: + bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) + bot.upload_image(debug=False) + return 0 + +class Tkdialog: + def __init__(self, photoDescription, photoUrl, filename): + self.root=Tk() + #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) + self.root.geometry("%ix%i+10-10"%(config.tkhorsize, config.tkvertsize)) + + self.root.title(filename) + self.photoDescription = photoDescription + self.filename = filename + self.photoUrl = photoUrl + self.skip=False + self.exit=False + + ## Init of the widgets + # The image + self.image=self.getImage(self.photoUrl, 800, 600) + self.imagePanel=Label(self.root, image=self.image) + + self.imagePanel.image = self.image + + # The filename + self.filenameLabel=Label(self.root,text=u"Suggested filename") + self.filenameField=Entry(self.root, width=100) + self.filenameField.insert(END, filename) + + # The description + self.descriptionLabel=Label(self.root,text=u"Suggested description") + self.descriptionScrollbar=Scrollbar(self.root, orient=VERTICAL) + self.descriptionField=Text(self.root) + self.descriptionField.insert(END, photoDescription) + self.descriptionField.config(state=NORMAL, height=12, width=100, padx=0, pady=0, wrap=WORD, yscrollcommand=self.descriptionScrollbar.set) + self.descriptionScrollbar.config(command=self.descriptionField.yview) + + # The buttons + self.okButton=Button(self.root, text="OK", command=self.okFile) + self.skipButton=Button(self.root, text="Skip", command=self.skipFile) + + ## Start grid + + # The image + self.imagePanel.grid(row=0, column=0, rowspan=11, columnspan=4) + + # The buttons + self.okButton.grid(row=11, column=1, rowspan=2) + self.skipButton.grid(row=11, column=2, rowspan=2) + + # The filename + self.filenameLabel.grid(row=13, column=0) + self.filenameField.grid(row=13, column=1, columnspan=3) + + # The description + self.descriptionLabel.grid(row=14, column=0) + self.descriptionField.grid(row=14, column=1, columnspan=3) + self.descriptionScrollbar.grid(row=14, column=5) + + def getImage(self, url, width, height): + image=urllib.urlopen(url).read() + output = StringIO.StringIO(image) + image2 = Image.open(output) + image2.thumbnail((width, height)) + imageTk = ImageTk.PhotoImage(image2) + return imageTk + + def okFile(self): + ''' + The user pressed the OK button. + ''' + self.filename=self.filenameField.get() + self.photoDescription=self.descriptionField.get(0.0, END) + self.root.destroy() + + def skipFile(self): + ''' + The user pressed the Skip button. + ''' + self.skip=True + self.root.destroy() + + def run(self): + ''' + Activate the dialog and return the new name and if the image is skipped. + ''' + self.root.mainloop() + return (self.photoDescription, self.filename, self.skip) + + +def usage(): + wikipedia.output(u"Flickrripper is a tool to transfer flickr photos to Wikimedia Commons") + wikipedia.output(u"-group_id:<group_id>\n") + wikipedia.output(u"-photoset_id:<photoset_id>\n") + wikipedia.output(u"-user_id:<user_id>\n") + wikipedia.output(u"-tags:<tag>\n") + return + +def main(): + site = wikipedia.getSite(u'commons', u'commons') + wikipedia.setSite(site) + #imagerecat.initLists() + + #Get the api key + if(config.flickr['api_key']): + flickr = flickrapi.FlickrAPI(config.flickr['api_key']) + else: + wikipedia.output('Flickr api key not found! Get yourself an api key') + wikipedia.output('Any flickr user can get a key at http://www.flickr.com/services/api/keys/apply/') + return + + group_id = u'' + photoset_id = u'' + user_id = u'' + tags = u'' + totalPhotos = 0 + uploadedPhotos = 0 + + # Do we mark the images as reviewed right away? + if config.flickr['review']: + flickrreview = config.flickr['review'] + else: + flickrreview = False + + # Set the Flickr reviewer + if config.flickr['reviewer']: + reviewer = config.flickr['reviewer'] + elif 'commons' in config.sysopnames['commons']: + print config.sysopnames['commons'] + reviewer = config.sysopnames['commons']['commons'] + elif 'commons' in config.usernames['commons']: + reviewer = config.usernames['commons']['commons'] + else: + reviewer = u'' + + override = u'' + + for arg in wikipedia.handleArgs(): + if arg.startswith('-group_id'): + if len(arg) == 9: + group_id = wikipedia.input(u'What is the group_id of the pool?') + else: + group_id = arg[10:] + elif arg.startswith('-photoset_id'): + if len(arg) == 12: + photoset_id = wikipedia.input(u'What is the photoset_id)?') + else: + photoset_id = arg[13:] + elif arg.startswith('-user_id'): + if len(arg) == 8: + user_id = wikipedia.input(u'What is the user_id of the flickr user?') + else: + user_id = arg[9:] + elif arg.startswith('-tags'): + if len(arg) == 5: + tags = wikipedia.input(u'What is the tag you want to filter out (currently only one supported)?') + else: + tags = arg[6:] + elif arg == '-flickrreview': + flickrreview = True + elif arg.startswith('-reviewer'): + if len(arg) == 9: + reviewer = wikipedia.input(u'Who is the reviewer?') + else: + reviewer = arg[10:] + elif arg.startswith('-override'): + if len(arg) == 9: + override = wikipedia.input(u'What is the override text?') + else: + override = arg[10:] + + if user_id or group_id or photoset_id: + for photo_id in getPhotos(flickr=flickr, user_id=user_id, group_id=group_id, photoset_id=photoset_id, tags=tags): + uploadedPhotos = uploadedPhotos + processPhoto(flickr=flickr, photo_id=photo_id, flickrreview=flickrreview, reviewer=reviewer, override=override) + totalPhotos = totalPhotos + 1 + else: + usage() + + wikipedia.output(u'Finished running') + wikipedia.output(u'Total photos: ' + str(totalPhotos)) + wikipedia.output(u'Uploaded photos: ' + str(uploadedPhotos)) + +if __name__ == "__main__": + try: + main() + finally: + wikipedia.stopme() Property changes on: trunk/pywikipedia/flickrripper.py ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native

1 0

SVN: [7187] trunk/pywikipedia
by a_engels＠svn.wikimedia.org 28 Aug '09

28 Aug '09

Revision: 7187 Author: a_engels Date: 2009-08-28 23:11:46 +0000 (Fri, 28 Aug 2009) Log Message: ----------- 2 changes to flickrripper: 1. Put the buttons 'OK' and 'Skip' directly under the image to avoid having them fall off the screen; 2. Make screen size a configuration variable Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2009-08-28 21:29:48 UTC (rev 7186) +++ trunk/pywikipedia/config.py 2009-08-28 23:11:46 UTC (rev 7187) @@ -146,6 +146,11 @@ except: colorized_output = False +# An indication of the size of your screen, or rather the size of the screen +# to be shown, for flickrripper +tkhorsize = 1600 +tkvertsize = 1000 + ############## EXTERNAL EDITOR SETTINGS ############## # The command for the editor you want to use. If set to None, a simple Tkinter # editor will be used. Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-28 21:29:48 UTC (rev 7186) +++ trunk/pywikipedia/flickrripper.py 2009-08-28 23:11:46 UTC (rev 7187) @@ -211,7 +211,7 @@ def __init__(self, photoDescription, photoUrl, filename): self.root=Tk() #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) - self.root.geometry("1600x1000+10-10") + self.root.geometry("%ix%i+10-10"%(config.tkhorsize, config.tkvertsize)) self.root.title(filename) self.photoDescription = photoDescription @@ -245,22 +245,23 @@ self.skipButton=Button(self.root, text="Skip", command=self.skipFile) ## Start grid + # The image self.imagePanel.grid(row=0, column=0, rowspan=11, columnspan=4) - + + # The buttons + self.okButton.grid(row=11, column=1, rowspan=2) + self.skipButton.grid(row=11, column=2, rowspan=2) + # The filename - self.filenameLabel.grid(row=11, column=0) - self.filenameField.grid(row=11, column=1, columnspan=3) + self.filenameLabel.grid(row=13, column=0) + self.filenameField.grid(row=13, column=1, columnspan=3) # The description - self.descriptionLabel.grid(row=12, column=0) - self.descriptionField.grid(row=12, column=1, columnspan=3) - self.descriptionScrollbar.grid(row=12, column=5) + self.descriptionLabel.grid(row=14, column=0) + self.descriptionField.grid(row=14, column=1, columnspan=3) + self.descriptionScrollbar.grid(row=14, column=5) - # The buttons - self.okButton.grid(row=13, column=1, rowspan=2) - self.skipButton.grid(row=13, column=2, rowspan=2) - def getImage(self, url, width, height): image=urllib.urlopen(url).read() output = StringIO.StringIO(image)

1 0

SVN: [7186] trunk/pywikipedia
by a_engels＠svn.wikimedia.org 28 Aug '09

28 Aug '09

Revision: 7186 Author: a_engels Date: 2009-08-28 21:29:48 +0000 (Fri, 28 Aug 2009) Log Message: ----------- enable configuring flickrripper Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2009-08-28 20:52:34 UTC (rev 7185) +++ trunk/pywikipedia/config.py 2009-08-28 21:29:48 UTC (rev 7186) @@ -330,6 +330,13 @@ # http://search.msn.com/developer msn_appid = '' +# Using the Flickr api +flickr = { + 'api_key': None, # Provide your key! + 'review': False, # Do we use automatically make our uploads reviewed? + 'reviewer': None, # If so, under what reviewer name? + } + ############## COPYRIGHT SETTINGS ############## # Enable/disable search engine in copyright.py script Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-28 20:52:34 UTC (rev 7185) +++ trunk/pywikipedia/flickrripper.py 2009-08-28 21:29:48 UTC (rev 7186) @@ -34,10 +34,10 @@ import sys, urllib, re, StringIO import wikipedia, config, query, imagerecat, upload -import flickrapi +import flickrapi # see: http://stuvel.eu/projects/flickrapi import xml.etree.ElementTree from Tkinter import * -from PIL import Image, ImageTk +from PIL import Image, ImageTk # see: http://www.pythonware.com/products/pil/ def getPhoto(flickr = None, photo_id = ''): ''' @@ -329,9 +329,10 @@ # Set the Flickr reviewer if config.flickr['reviewer']: reviewer = config.flickr['reviewer'] - elif config.sysopnames['commons']['commons']: + elif 'commons' in config.sysopnames['commons']: + print config.sysopnames['commons'] reviewer = config.sysopnames['commons']['commons'] - elif config.usernames['commons']['commons']: + elif 'commons' in config.usernames['commons']: reviewer = config.usernames['commons']['commons'] else: reviewer = u''

1 0

SVN: [7185] trunk/pywikipedia/interwiki.py
by a_engels＠svn.wikimedia.org 28 Aug '09

28 Aug '09

Revision: 7185 Author: a_engels Date: 2009-08-28 20:52:34 +0000 (Fri, 28 Aug 2009) Log Message: ----------- adding command-line option -lack, to only work on pages lacking an interwiki to a specified language. Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-08-28 17:09:46 UTC (rev 7184) +++ trunk/pywikipedia/interwiki.py 2009-08-28 20:52:34 UTC (rev 7185) @@ -81,6 +81,11 @@ -skipauto use to skip all pages that can be translated automatically, like dates, centuries, months, etc. (note: without ending colon) + -lack: used as -lack:xx with xx a language code: only work on pages + without links to language xx. You can also add a number nn + lick -lack:xx:nn, so that the bot only works on pages with + at least n interwiki links (the default value for n is 1). + These arguments are useful to provide hints to the bot: -hint: used as -hint:de:Anweisung to give the robot a hint @@ -506,6 +511,8 @@ nobackonly = False hintsareright = False contentsondisk = config.interwiki_contents_on_disk + lacklanguage = None + minlinks = 0 class StoredPage(wikipedia.Page): """ @@ -745,6 +752,7 @@ self.untranslated = None self.hintsAsked = False self.forcedStop = False + self.workonme = True def getFoundDisambig(self, site): """ @@ -1007,6 +1015,9 @@ f.close() def askForHints(self, counter): + if not self.workonme: + # Do not ask hints for pages that we don't work on anyway + return if (self.untranslated or globalvar.askhints) and not self.hintsAsked and not self.originPage.isRedirectPage(): # Only once! self.hintsAsked = True @@ -1138,6 +1149,13 @@ if globalvar.untranslatedonly: # Ignore the interwiki links. iw = () + if globalvar.lacklanguage: + if globalvar.lacklanguage in [link.site().language() for link in iw]: + iw = () + self.workonme = False + if len(iw) < globalvar.minlinks: + iw = () + self.workonme = False elif globalvar.autonomous and duplicate: @@ -1324,6 +1342,8 @@ be told to make another get request first.""" if not self.isDone(): raise "Bugcheck: finish called before done" + if not self.workonme: + return if self.forcedStop: wikipedia.output(u"======Aborted processing %s======" % self.originPage.aslink(True)) return @@ -2080,6 +2100,13 @@ globalvar.minsubjects = int(arg[7:]) elif arg.startswith('-query:'): globalvar.maxquerysize = int(arg[7:]) + elif arg.startswith('-lack:'): + remainder = arg[6:].split(':') + globalvar.lacklanguage = remainder[0] + if len(remainder) > 1: + globalvar.minlinks = int(remainder[1]) + else: + globalvar.minlinks = 1 elif arg == '-back': globalvar.nobackonly = True else:

1 0

SVN: [7184] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 28 Aug '09

28 Aug '09

Revision: 7184 Author: alexsh Date: 2009-08-28 17:09:46 +0000 (Fri, 28 Aug 2009) Log Message: ----------- Patch uk messages in #2843716 #2838315 and vec in #2840076 Modified Paths: -------------- trunk/pywikipedia/interwiki.py trunk/pywikipedia/replace.py trunk/pywikipedia/template.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-08-28 17:07:51 UTC (rev 7183) +++ trunk/pywikipedia/interwiki.py 2009-08-28 17:09:46 UTC (rev 7184) @@ -461,6 +461,7 @@ 'th': (u'โรบอต ', u'เพิ่ม', u'ลบ', u'แก้ไข'), 'uk': (u'робот ', u'додав', u'видалив', u'змінив'), 'uz': (u'Bot ', u'Qoʻshdi', u'Tuzatdi', u'Oʻchirdi'), + 'vec': (u'Bot: ', u'Zonto', u'Cavo', u'Canbio'), 'vi': (u'robot ', u'Thêm', u'Dời', u'Thay'), 'vo': (u'bot ', u'läükon', u'moükon', u'votükon'), 'war':(u'robot ', u'Gindugngan', u'Gintanggal', u'Ginliwat'), Modified: trunk/pywikipedia/replace.py =================================================================== --- trunk/pywikipedia/replace.py 2009-08-28 17:07:51 UTC (rev 7183) +++ trunk/pywikipedia/replace.py 2009-08-28 17:09:46 UTC (rev 7184) @@ -168,6 +168,7 @@ 'ru': u'Робот: Автоматизированная замена текста %s', 'sr': u'Бот: Аутоматска замена текста %s', 'sv': u'Bot: Automatisk textersättning: %s', + 'uk': u'Бот: Автоматизована заміна тексту: %s', 'zh': u'機器人:執行文字代換作業 %s', } Modified: trunk/pywikipedia/template.py =================================================================== --- trunk/pywikipedia/template.py 2009-08-28 17:07:51 UTC (rev 7183) +++ trunk/pywikipedia/template.py 2009-08-28 17:09:46 UTC (rev 7184) @@ -162,6 +162,7 @@ 'pt':u'Bot: Alterando predefinição: %s', 'ru':u'Робот: замена шаблона: %s', 'sr':u'Бот: Измена шаблона: %s', + 'uk':u'Робот: заміна шаблону: %s', 'zh':u'機器人: 更改模板 %s', } @@ -183,6 +184,7 @@ 'pl':u'Robot zmienia szablony: %s', 'pt':u'Bot: Alterando predefinição: %s', 'ru':u'Робот: замена шаблонов: %s', + 'uk':u'Робот: заміна шаблонів: %s', 'zh':u'機器人: 更改模板 %s', } @@ -207,6 +209,7 @@ 'pt':u'Bot: Removendo predefinição: %s', 'ru':u'Робот: удаление шаблона: %s', 'sr':u'Бот: Уклањање шаблона: %s', + 'uk':u'Робот: видалення шаблону: %s', 'zh':u'機器人: 移除模板 %s', } @@ -228,6 +231,7 @@ 'pl':u'Robot usuwa szablony: %s', 'ru':u'Робот: удаление шаблонов: %s', 'pt':u'Bot: Removendo predefinição: %s', + 'uk':u'Робот: видалення шаблонів: %s', 'zh':u'機器人: 移除模板 %s', }

1 0

SVN: [7183] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 28 Aug '09

28 Aug '09

Revision: 7183 Author: alexsh Date: 2009-08-28 17:07:51 +0000 (Fri, 28 Aug 2009) Log Message: ----------- typo Modified Paths: -------------- trunk/pywikipedia/checkimages.py trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-08-27 13:02:48 UTC (rev 7182) +++ trunk/pywikipedia/checkimages.py 2009-08-28 17:07:51 UTC (rev 7183) @@ -256,10 +256,10 @@ 'de' :[u'ABFbot'], 'en' :[u'OrphanBot'], 'it' :[u'Filbot', u'Nikbot', u'.snoopyBot.'], - 'ja' :[u'alexbot'], + 'ja' :[u'Alexbot'], 'ko' :[u'Kwjbot IV'], 'ta' :[u'TrengarasuBOT'], - 'zh' :[u'alexbot'], + 'zh' :[u'Alexbot'], } # The message that the bot will add the second time that find another license problem. Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-27 13:02:48 UTC (rev 7182) +++ trunk/pywikipedia/flickrripper.py 2009-08-28 17:07:51 UTC (rev 7183) @@ -39,7 +39,7 @@ from Tkinter import * from PIL import Image, ImageTk -def getPhoto(flickr=None, photo_id=''): +def getPhoto(flickr = None, photo_id = ''): ''' Get the photo info and the photo sizes so we can use these later on ''' @@ -49,19 +49,19 @@ #xml.etree.ElementTree.dump(photoSizes) return (photoInfo, photoSizes) -def isAllowedLicense(photoInfo=None): +def isAllowedLicense(photoInfo = None): ''' Check if the image contains the right license ''' license = photoInfo.find('photo').attrib['license'] - if (license=='4' or license=='5'): + if license == '4' or license == '5': #Is cc-by or cc-by-sa return True else: #We don't accept other licenses return False -def getTags(photoInfo=None): +def getTags(photoInfo = None): ''' Get all the tags on a photo ''' @@ -71,7 +71,7 @@ return result -def getFlinfoDescription(photo_id=0): +def getFlinfoDescription(photo_id = 0): ''' Get the description from http://wikipedia.ramselehof.de/flinfo.php ''' @@ -321,7 +321,7 @@ uploadedPhotos = 0 # Do we mark the images as reviewed right away? - if(config.flickr['review']): + if config.flickr['review']: flickrreview = config.flickr['review'] else: flickrreview = False @@ -372,7 +372,7 @@ else: override = arg[10:] - if(user_id or group_id or photoset_id): + if user_id or group_id or photoset_id: for photo_id in getPhotos(flickr=flickr, user_id=user_id, group_id=group_id, photoset_id=photoset_id, tags=tags): uploadedPhotos = uploadedPhotos + processPhoto(flickr=flickr, photo_id=photo_id, flickrreview=flickrreview, reviewer=reviewer, override=override) totalPhotos = totalPhotos + 1

1 0

SVN: [7182] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 27 Aug '09

27 Aug '09

Revision: 7182 Author: filnik Date: 2009-08-27 13:02:48 +0000 (Thu, 27 Aug 2009) Log Message: ----------- Bugfix: when report for a duplicate, now it will catch if the same image has been reported or not Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-08-27 12:47:00 UTC (rev 7181) +++ trunk/pywikipedia/checkimages.py 2009-08-27 13:02:48 UTC (rev 7182) @@ -953,7 +953,7 @@ if len(images_to_tag_list) != 0 and not only_report: already_reported_in_past = self.countEdits(u'File:%s' % images_to_tag_list[-1], self.botolist) image_to_resub = images_to_tag_list[-1] - from_regex = r'\n\*\[\[:%s\]\]' % re.escape(self.image_namespace + image_to_resub) + from_regex = r'\n\*\[\[:File:%s\]\]' % re.escape(self.convert_to_url(self.imageName)) # Delete the image in the list where we're write on text_for_the_report = re.sub(from_regex, '', text_for_the_report) # if you want only one edit, the edit found should be more than 0 -> num - 1

1 0

SVN: [7181] trunk/pywikipedia/blockpageschecker.py
by filnik＠svn.wikimedia.org 27 Aug '09

27 Aug '09

Revision: 7181 Author: filnik Date: 2009-08-27 12:47:00 +0000 (Thu, 27 Aug 2009) Log Message: ----------- Adding new feature: on it.wikipedia the template used is now only one for all the protection. I maintain the old settings for backward compatibily and because the old templates are still used (but deprecated) Modified Paths: -------------- trunk/pywikipedia/blockpageschecker.py Modified: trunk/pywikipedia/blockpageschecker.py =================================================================== --- trunk/pywikipedia/blockpageschecker.py 2009-08-27 11:20:54 UTC (rev 7180) +++ trunk/pywikipedia/blockpageschecker.py 2009-08-27 12:47:00 UTC (rev 7181) @@ -54,7 +54,7 @@ """ # # (C) Monobi a.k.a. Wikihermit, 2007 -# (C) Filnik, 2007-2008 +# (C) Filnik, 2007-2008-2009 # (C) NicDumZ, 2008 # # Distributed under the terms of the MIT license. @@ -75,9 +75,6 @@ #--------------------- PREFERENCES -------------------# ################### -- Edit below! -- ################# -# Added a new feature! Please update and add the settings in order -# to improve the intelligence of this script ;-) -# Regex to get the semi-protection template templateSemiProtection = { 'en': None, 'it':[r'\{\{(?:[Tt]emplate:|)[Aa]vvisobloccoparziale(?:|[ _]scad\|.*?|\|.*?)\}\}', @@ -110,12 +107,21 @@ 'ja':[ur'(?<!\<nowiki\>)\{\{(?:[Tt]emplate:|)移動保護(?:[Ss]|)(?:\|.+|)\}\}(?!\<\/nowiki\>)\s*(?:\r\n|)*'], 'zh':[ur'\{\{(?:[Tt]emplate:|)Protected|(?:[Mm]|[Mm]ove|移[動动])(?:\|.+|)\}\}(\n+?|)',ur'\{\{(?:[Tt]emplate:|)Mini-protected|(?:[Mm]|[Mm]ove|移[動动])(?:\|.+|)\}\}(\n+?|)',ur'\{\{(?:[Tt]emplate:|)Protected-logo|(?:[Mm]|[Mm]ove|移[動动])(?:\|.+|)\}\}(\n+?|)'], } -# Array: 0 => Semi-block, 1 => Total Block, 2 => Semi-Move, 3 => Total-Move + +# If you use only one template for all the type of protection, put it here. +# You may use only one template or an unique template and some other "old" template that the +# script should still check (as on it.wikipedia) +templateUnique = { + 'en': None, + 'it': [r'\{\{(?:[Tt]emplate:|)[Pp]rotetta\}\}'], +} + +# Array: 0 => Semi-block, 1 => Total Block, 2 => Semi-Move, 3 => Total-Move, 4 => template-unique templateNoRegex = { - 'it':['{{Avvisobloccoparziale}}', '{{Avvisoblocco}}', None, None], - 'fr':['{{Semi-protection}}', '{{Protection}}', None, None], - 'ja':[u'{{半保護}}', u'{{保護}}', u'{{移動半保護}}', u'{{移動保護}}'], - 'zh':[u'{{Protected/semi}}',u'{{Protected}}',u'{{Protected/ms}}',u'{{Protected/move}}'], + 'it':['{{Avvisobloccoparziale}}', '{{Avvisoblocco}}', None, None, '{{Protetta}}'], + 'fr':['{{Semi-protection}}', '{{Protection}}', None, None, None], + 'ja':[u'{{半保護}}', u'{{保護}}', u'{{移動半保護}}', u'{{移動保護}}', None], + 'zh':[u'{{Protected/semi}}',u'{{Protected}}',u'{{Protected/ms}}',u'{{Protected/move}}', None], } # Category where the bot will check @@ -124,7 +130,7 @@ 'ar':[u'تصنيف:محتويات محمية'], 'fr':[u'Category:Page semi-protégée', u'Category:Page protégée', u'Catégorie:Article protégé'], 'he':[u'קטגוריה:ויקיפדיה: דפים מוגנים', u'קטגוריה:ויקיפדיה: דפים מוגנים חלקית'], - 'it':[u'Categoria:Pagine semiprotette', u'Categoria:Voci_protette'], + 'it':[u'Categoria:Pagine semiprotette', u'Categoria:Voci protette', u'Categoria:Pagine protette - scadute'], 'ja':[u'Category:編集保護中の記事',u'Category:編集半保護中の記事', u'Category:移動保護中の記事',], 'pt':[u'Category:!Páginas protegidas', u'Category:!Páginas semiprotegidas'], @@ -149,7 +155,7 @@ #------------------ END PREFERENCES ------------------# ################## -- Edit above! -- ################## -def understandBlock(text, TTP, TSP, TSMP, TTMP): +def understandBlock(text, TTP, TSP, TSMP, TTMP, TU): """ Understand if the page is blocked and if it has the right template """ for catchRegex in TTP: # TTP = templateTotalProtection resultCatch = re.findall(catchRegex, text) @@ -159,6 +165,10 @@ resultCatch = re.findall(catchRegex, text) if resultCatch: return ('autoconfirmed-total', catchRegex) + for catchRegex in TU: + resultCatch = re.findall(catchRegex, text) + if resultCatch: + return ('unique', catchRegex) if TSMP != None and TTMP != None and TTP != TTMP and TSP != TSMP: for catchRegex in TTMP: resultCatch = re.findall(catchRegex, text) @@ -235,6 +245,7 @@ TSMP = wikipedia.translate(site, templateSemiMoveProtection) TTMP = wikipedia.translate(site, templateTotalMoveProtection) TNR = wikipedia.translate(site, templateNoRegex) + TU = wikipedia.translate(site, templateUnique) category = wikipedia.translate(site, categoryToCheck) commentUsed = wikipedia.translate(site, comment) @@ -285,7 +296,7 @@ # Understand, according to the template in the page, what should be the protection # and compare it with what there really is. - TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP) + TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP, TU) # Only to see if the text is the same or not... oldtext = text # keep track of the changes for each step (edit then move) @@ -294,31 +305,37 @@ if not editRestr: # page is not edit-protected # Deleting the template because the page doesn't need it. - replaceToPerform = u'|'.join(TTP + TSP) + replaceToPerform = u'|'.join(TTP + TSP + TU) text, changes = re.subn('(?:<noinclude>|)(%s)(?:</noinclude>|)' % replaceToPerform, '', text) wikipedia.output(u'The page is editable for all, deleting the template...') elif editRestr[0] == 'sysop': # total edit protection - if TemplateInThePage[0] == 'sysop-total' and TTP != None: + if (TemplateInThePage[0] == 'sysop-total' and TTP != None) or (TemplateInThePage[0] == 'unique' and TU != None): msg = 'The page is protected to the sysop' if not moveBlockCheck: msg += ', skipping...' wikipedia.output(msg) else: wikipedia.output(u'The page is protected to the sysop, but the template seems not correct. Fixing...') - text, changes = re.subn(TemplateInThePage[1], TNR[1], text) + if TU != None: + text, changes = re.subn(TemplateInThePage[1], TNR[4], text) + else: + text, changes = re.subn(TemplateInThePage[1], TNR[1], text) - elif TSP != None: + elif TSP != None or TU != None: # implicitely editRestr[0] = 'autoconfirmed', edit-Semi-protection - if TemplateInThePage[0] == 'autoconfirmed-total': + if TemplateInThePage[0] == 'autoconfirmed-total' or TemplateInThePage[0] == 'unique': msg = 'The page is editable only for the autoconfirmed users' if not moveBlockCheck: msg += ', skipping...' wikipedia.output(msg) else: wikipedia.output(u'The page is editable only for the autoconfirmed users, but the template seems not correct. Fixing...') - text, changes = re.subn(TemplateInThePage[1], TNR[0], text) + if TU != None: + text, changes = re.subn(TemplateInThePage[1], TNR[4], text) + else: + text, changes = re.subn(TemplateInThePage[1], TNR[0], text) if changes == 0: # We tried to fix edit-protection templates, but it did not work. @@ -332,23 +349,29 @@ if not moveRestr: wikipedia.output(u'The page is movable for all, deleting the template...') # Deleting the template because the page doesn't need it. - replaceToPerform = u'|'.join(TSMP + TTMP) + replaceToPerform = u'|'.join(TSMP + TTMP + TU) text, changes = re.subn('(?:<noinclude>|)(%s)(?:</noinclude>|)' % replaceToPerform, '', text) elif moveRestr[0] == 'sysop': # move-total-protection - if TemplateInThePage[0] == 'sysop-move' and TTMP != None: + if (TemplateInThePage[0] == 'sysop-move' and TTMP != None) or (TemplateInThePage[0] == 'unique' and TU != None): wikipedia.output(u'The page is protected from moving to the sysop, skipping...') else: wikipedia.output(u'The page is protected from moving to the sysop, but the template seems not correct. Fixing...') + if TU != None: + text, changes = re.subn(TemplateInThePage[1], TNR[4], text) + else: text, changes = re.subn(TemplateInThePage[1], TNR[3], text) - elif TSMP != None: + elif TSMP != None or TU != None: # implicitely moveRestr[0] = 'autoconfirmed', move-semi-protection - if TemplateInThePage[0] == 'autoconfirmed-move': + if TemplateInThePage[0] == 'autoconfirmed-move' or TemplateInThePage[0] == 'unique': wikipedia.output(u'The page is movable only for the autoconfirmed users, skipping...') else: wikipedia.output(u'The page is movable only for the autoconfirmed users, but the template seems not correct. Fixing...') + if TU != None: + text, changes = re.subn(TemplateInThePage[1], TNR[4], text) + else: text, changes = re.subn(TemplateInThePage[1], TNR[2], text) if changes == 0:

1 0

SVN: [7180] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 27 Aug '09

27 Aug '09

Revision: 7180 Author: filnik Date: 2009-08-27 11:20:54 +0000 (Thu, 27 Aug 2009) Log Message: ----------- bugfix, I forgot a self Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-08-27 10:39:01 UTC (rev 7179) +++ trunk/pywikipedia/checkimages.py 2009-08-27 11:20:54 UTC (rev 7180) @@ -546,6 +546,7 @@ duplicatesReport = False, logFullError = True): """ Constructor, define some global variable """ self.site = site + self.logFullError = logFullError self.logFulNumber = logFulNumber self.settings = wikipedia.translate(self.site, page_with_settings) self.rep_page = wikipedia.translate(self.site, report_page) @@ -992,10 +993,11 @@ except wikipedia.IsRedirectPage: text_get = another_page.getRedirectTarget().get() if len(text_get) >= self.logFulNumber: - if logFullError: + if self.logFullError: raise LogIsFull(u"The log page (%s) is full! Please delete the old files reported." % another_page.title()) else: wikipedia.output(u"The log page (%s) is full! Please delete the old files reported. Skip!" % another_page.title()) + return True # Don't report, but continue with the check (we don't now if this is the first time we check this file or not) pos = 0 # The talk page includes "_" between the two names, in this way i replace them to " " n = re.compile(regex, re.UNICODE|re.DOTALL)

1 0

SVN: [7179] trunk/pywikipedia/wikipedia.py
by filnik＠svn.wikimedia.org 27 Aug '09

27 Aug '09

Revision: 7179 Author: filnik Date: 2009-08-27 10:39:01 +0000 (Thu, 27 Aug 2009) Log Message: ----------- Bugfix: a coma is in reality a dot. This blocked checkimages.py (that uses a function of wikipedia.py) before it can warn the user Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-26 21:12:43 UTC (rev 7178) +++ trunk/pywikipedia/wikipedia.py 2009-08-27 10:39:01 UTC (rev 7179) @@ -3028,7 +3028,7 @@ 'titles' :self.title(), } try: - data = query.GetData(params, self,site(), encodeTitle = False)['query']['pages'] + data = query.GetData(params, self.site(), encodeTitle = False)['query']['pages'] except KeyError: raise NoPage(u'API Error, nothing found in the APIs')

1 0

SVN: [7178] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 26 Aug '09

26 Aug '09

Revision: 7178 Author: filnik Date: 2009-08-26 21:12:43 +0000 (Wed, 26 Aug 2009) Log Message: ----------- New functionality: if given, the bot won't rise an error when the log is full Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-08-26 20:59:03 UTC (rev 7177) +++ trunk/pywikipedia/checkimages.py 2009-08-26 21:12:43 UTC (rev 7178) @@ -44,6 +44,8 @@ -untagged[:#] - Use daniel's tool as generator ( http://toolserver.org/~daniel/WikiSense/UntaggedImages.php ) + -nologerror - If given, this option will disable the error that is risen when the log is full. + ---- Istructions for the real-time settings ---- * For every new block you have to add: @@ -541,7 +543,7 @@ # Here there is the main class. class main: def __init__(self, site, logFulNumber = 25000, sendemailActive = False, - duplicatesReport = False): + duplicatesReport = False, logFullError = True): """ Constructor, define some global variable """ self.site = site self.logFulNumber = logFulNumber @@ -990,7 +992,10 @@ except wikipedia.IsRedirectPage: text_get = another_page.getRedirectTarget().get() if len(text_get) >= self.logFulNumber: - raise LogIsFull(u"The log page (%s) is full! Please delete the old files reported." % another_page.title()) + if logFullError: + raise LogIsFull(u"The log page (%s) is full! Please delete the old files reported." % another_page.title()) + else: + wikipedia.output(u"The log page (%s) is full! Please delete the old files reported. Skip!" % another_page.title()) pos = 0 # The talk page includes "_" between the two names, in this way i replace them to " " n = re.compile(regex, re.UNICODE|re.DOTALL) @@ -1497,6 +1502,7 @@ duplicatesActive = False # Use the duplicate option duplicatesReport = False # Use the duplicate-report option sendemailActive = False # Use the send-email + logFullError = True # Raise an error when the log is full # Here below there are the parameters. for arg in wikipedia.handleArgs(): @@ -1512,6 +1518,8 @@ time_sleep = int(arg[6:]) elif arg == '-break': repeat = False + elif arg == '-nologerror': + logFullError = False elif arg == '-commons': commonsActive = True elif arg.startswith('-duplicates'): @@ -1623,7 +1631,8 @@ # Main Loop while 1: # Defing the Main Class. - mainClass = main(site, sendemailActive = sendemailActive, duplicatesReport = duplicatesReport) + mainClass = main(site, sendemailActive = sendemailActive, + duplicatesReport = duplicatesReport, logFullError = logFullError) # Untagged is True? Let's take that generator if untagged == True: generator = mainClass.untaggedGenerator(projectUntagged, limit)

1 0

SVN: [7177] trunk/pywikipedia/add_text.py
by filnik＠svn.wikimedia.org 26 Aug '09

26 Aug '09

Revision: 7177 Author: filnik Date: 2009-08-26 20:59:03 +0000 (Wed, 26 Aug 2009) Log Message: ----------- Bugfix by mauro742 for Bug ID: 2839738 Modified Paths: -------------- trunk/pywikipedia/add_text.py Modified: trunk/pywikipedia/add_text.py =================================================================== --- trunk/pywikipedia/add_text.py 2009-08-26 17:50:13 UTC (rev 7176) +++ trunk/pywikipedia/add_text.py 2009-08-26 20:59:03 UTC (rev 7177) @@ -164,7 +164,7 @@ result = re.findall(regexSkip, text) if result != []: wikipedia.output(u'Exception! regex (or word) used with -except is in the page. Skip!') - return (False, always) # continue + return (False, False, always) # continue # If not up, text put below if not up: newtext = text @@ -215,7 +215,7 @@ if choice == 'a': always = True if choice == 'n': - return (False, always) + return (False, False, always) if choice == 'y' or always: try: if always: @@ -224,7 +224,7 @@ page.put_async(newtext, summary) except wikipedia.EditConflict: wikipedia.output(u'Edit conflict! skip!') - return (False, always) + return (False, False, always) except wikipedia.ServerError: errorCount += 1 if errorCount < 5: @@ -241,11 +241,11 @@ return (False, always) except wikipedia.LockedPage: wikipedia.output(u'Skipping %s (locked page)' % page.title()) - return (False, always) + return (False, False, always) else: # Break only if the errors are one after the other... errorCount = 0 - return (True, always) + return (True, True, always) else: return (text, newtext, always)

1 0

SVN: [7176] branches/rewrite/pywikibot
by russblau＠svn.wikimedia.org 26 Aug '09

26 Aug '09

Revision: 7176 Author: russblau Date: 2009-08-26 17:50:13 +0000 (Wed, 26 Aug 2009) Log Message: ----------- Merge family file changes from trunk Modified Paths: -------------- branches/rewrite/pywikibot/families/commons_family.py branches/rewrite/pywikibot/families/incubator_family.py branches/rewrite/pywikibot/families/mediawiki_family.py branches/rewrite/pywikibot/families/meta_family.py branches/rewrite/pywikibot/families/species_family.py branches/rewrite/pywikibot/families/test_family.py branches/rewrite/pywikibot/families/wikibooks_family.py branches/rewrite/pywikibot/families/wikinews_family.py branches/rewrite/pywikibot/families/wikipedia_family.py branches/rewrite/pywikibot/families/wikiquote_family.py branches/rewrite/pywikibot/families/wikisource_family.py branches/rewrite/pywikibot/families/wikiversity_family.py branches/rewrite/pywikibot/families/wiktionary_family.py branches/rewrite/pywikibot/family.py Property Changed: ---------------- branches/rewrite/pywikibot/families/ branches/rewrite/pywikibot/family.py Property changes on: branches/rewrite/pywikibot/families ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/pywikipedia/families:5896-6197,6212-7067 + /trunk/pywikipedia/families:5896-6197,6212-7067,7070-7175 Modified: branches/rewrite/pywikibot/families/commons_family.py =================================================================== --- branches/rewrite/pywikibot/families/commons_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/commons_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -39,7 +39,7 @@ } def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def dbName(self, code): return 'commonswiki_p' Modified: branches/rewrite/pywikibot/families/incubator_family.py =================================================================== --- branches/rewrite/pywikibot/families/incubator_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/incubator_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -15,7 +15,7 @@ } def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/mediawiki_family.py =================================================================== --- branches/rewrite/pywikibot/families/mediawiki_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/mediawiki_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -17,7 +17,7 @@ } def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/meta_family.py =================================================================== --- branches/rewrite/pywikibot/families/meta_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/meta_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -17,7 +17,7 @@ self.cross_allowed = ['meta',] def version(self,code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/species_family.py =================================================================== --- branches/rewrite/pywikibot/families/species_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/species_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -16,7 +16,7 @@ self.interwiki_forward = 'wikipedia' def version(self,code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/test_family.py =================================================================== --- branches/rewrite/pywikibot/families/test_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/test_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -18,7 +18,7 @@ self.langs['test'] = None def version(self, code): - return '1.13alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/wikibooks_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikibooks_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikibooks_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -11,16 +11,16 @@ self.name = 'wikibooks' self.languages_by_size = [ - 'en', 'de', 'fr', 'hu', 'pt', 'ja', 'es', 'nl', 'it', 'pl', - 'he', 'sq', 'fi', 'vi', 'ru', 'cs', 'hr', 'th', 'zh', 'da', - 'sv', 'mk', 'tr', 'sr', 'fa', 'tl', 'id', 'is', 'ca', 'ta', - 'ar', 'no', 'ko', 'eo', 'ka', 'simple', 'bg', 'gl', 'lt', 'ro', + 'en', 'de', 'fr', 'zh', 'no', 'hu', 'pt', 'ja', 'es', 'it', + 'nl', 'pl', 'sq', 'he', 'fi', 'vi', 'ru', 'cs', 'hr', 'da', + 'sv', 'mk', 'th', 'tr', 'sr', 'fa', 'tl', 'ca', 'id', 'is', + 'ta', 'ar', 'ko', 'eo', 'ka', 'simple', 'bg', 'gl', 'lt', 'ro', 'sk', 'la', 'el', 'ia', 'ang', 'sl', 'et', 'mr', 'cv', 'ur', - 'oc', 'ml', 'uk', 'ms', 'eu', 'fy', 'lv', 'hi', 'ie', 'tg', - 'bn', 'hy', 'af', 'az', 'te', 'pa', 'bs', 'ky', 'be', 'sa', - 'zh-min-nan', 'ast', 'cy', 'tt', 'mg', 'km', 'si', 'ku', 'co', 'sw', - 'tk', 'ne', 'als', 'uz', 'vo', 'su', 'na', 'mn', 'lb', 'kn', - 'ps', 'xh', 'kk', 'za', 'nds', 'wa', 'zu', 'my', + 'oc', 'ml', 'ms', 'uk', 'eu', 'fy', 'ie', 'hi', 'tg', 'bn', + 'hy', 'af', 'az', 'te', 'pa', 'bs', 'ky', 'be', 'sa', 'zh-min-nan', + 'ast', 'cy', 'tt', 'mg', 'si', 'ku', 'co', 'tk', 'sw', 'ne', + 'als', 'su', 'uz', 'vo', 'na', 'kn', 'mn', 'kk', 'za', 'nds', + 'zu', 'my', ] for lang in self.languages_by_size: @@ -96,7 +96,7 @@ self.cross_allowed = ['fa', 'fy', 'it', 'ru', 'simple', 'zh'] def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/wikinews_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikinews_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikinews_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -11,9 +11,9 @@ self.name = 'wikinews' self.languages_by_size = [ - 'en', 'pl', 'de', 'sr', 'it', 'fr', 'pt', 'es', 'zh', 'sv', - 'ja', 'ru', 'nl', 'he', 'fi', 'sd', 'ar', 'cs', 'no', 'uk', - 'ca', 'hu', 'ro', 'th', 'bs', 'bg', 'ta', + 'en', 'sr', 'pl', 'it', 'de', 'fr', 'pt', 'es', 'zh', 'sv', + 'ja', 'ru', 'nl', 'he', 'fi', 'ar', 'sd', 'cs', 'no', 'uk', + 'hu', 'ca', 'ro', 'th', 'bs', 'bg', 'ta', 'tr' ] for lang in self.languages_by_size: @@ -47,7 +47,7 @@ return 'utf-8' def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/wikipedia_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikipedia_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikipedia_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -11,32 +11,32 @@ self.name = 'wikipedia' self.languages_by_size = [ - 'en', 'de', 'fr', 'pl', 'ja', 'it', 'nl', 'pt', 'es', 'ru', - 'sv', 'zh', 'no', 'fi', 'ca', 'uk', 'tr', 'cs', 'hu', 'ro', - 'vo', 'eo', 'da', 'sk', 'id', 'ko', 'ar', 'he', 'vi', 'lt', - 'sl', 'sr', 'bg', 'et', 'fa', 'hr', 'new', 'simple', 'ht', 'nn', - 'gl', 'th', 'te', 'el', 'ms', 'eu', 'ceb', 'hi', 'mk', 'ka', - 'la', 'lb', 'bs', 'br', 'is', 'az', 'sq', 'cy', 'mr', 'bpy', - 'sh', 'tl', 'lv', 'pms', 'be-x-old', 'bn', 'jv', 'oc', 'ta', 'io', - 'be', 'an', 'nds', 'su', 'scn', 'nap', 'ku', 'af', 'fy', 'ast', - 'sw', 'wa', 'zh-yue', 'bat-smg', 'qu', 'ur', 'cv', 'ml', 'ksh', 'war', - 'tg', 'ga', 'vec', 'roa-tara', 'uz', 'gd', 'gu', 'pam', 'kn', 'mi', - 'yi', 'yo', 'lmo', 'glk', 'nah', 'zh-min-nan', 'co', 'hsb', 'roa-rup', 'als', - 'kk', 'ia', 'li', 'hy', 'gan', 'sah', 'tt', 'tk', 'sa', 'wuu', - 'nds-nl', 'fo', 'os', 'vls', 'fiu-vro', 'arz', 'nrm', 'bcl', 'am', 'rm', - 'pag', 'map-bms', 'dv', 'mn', 'gv', 'se', 'diq', 'fur', 'ne', 'sco', - 'bar', 'lij', 'nov', 'bh', 'mt', 'ilo', 'pi', 'mzn', 'zh-classical', 'km', - 'ug', 'csb', 'frp', 'lad', 'si', 'pdc', 'kw', 'ang', 'haw', 'sc', - 'mg', 'to', 'kv', 'szl', 'pa', 'ps', 'ie', 'my', 'gn', 'ln', - 'hif', 'stq', 'wo', 'jbo', 'arc', 'crh', 'tpi', 'ty', 'ext', 'cbk-zam', - 'ky', 'eml', 'zea', 'srn', 'ay', 'myv', 'hak', 'pap', 'ig', 'kg', - 'so', 'or', 'kab', 'lo', 'rmy', 'ba', 'ce', 'sm', 'udm', 'av', - 'ks', 'nv', 'dsb', 'cu', 'got', 'kaa', 'tet', 'sd', 'bo', 'mdf', - 'iu', 'na', 'bm', 'as', 'cdo', 'chr', 'ee', 'om', 'pnt', 'pih', - 'kl', 'zu', 'ab', 'ti', 'ts', 'ss', 'za', 'bi', 'cr', 'dz', - 'xal', 've', 'ch', 'ha', 'xh', 'tn', 'bug', 'st', 'bxr', 'ik', - 'rw', 'chy', 'tw', 'ak', 'fj', 'ny', 'ff', 'sn', 'sg', 'lbe', - 'rn', 'ki', 'lg', 'tum', 'ng', 'mhr', + 'en', 'de', 'fr', 'pl', 'ja', 'it', 'nl', 'es', 'pt', 'ru', + 'sv', 'zh', 'no', 'fi', 'ca', 'uk', 'hu', 'tr', 'cs', 'ro', + 'vo', 'eo', 'da', 'sk', 'id', 'ko', 'ar', 'vi', 'he', 'lt', + 'sr', 'sl', 'bg', 'et', 'fa', 'hr', 'simple', 'new', 'ht', 'nn', + 'gl', 'th', 'ms', 'el', 'te', 'eu', 'ceb', 'hi', 'mk', 'ka', + 'la', 'br', 'bs', 'lb', 'is', 'az', 'cy', 'sq', 'mr', 'sh', + 'bpy', 'lv', 'tl', 'pms', 'be-x-old', 'jv', 'bn', 'oc', 'ta', 'io', + 'be', 'an', 'scn', 'nds', 'su', 'nap', 'ku', 'sw', 'af', 'fy', + 'ast', 'zh-yue', 'bat-smg', 'wa', 'war', 'ur', 'qu', 'ml', 'ksh', 'cv', + 'tg', 'ga', 'vec', 'roa-tara', 'gu', 'gd', 'uz', 'pam', 'kn', 'yi', + 'mi', 'lmo', 'yo', 'zh-min-nan', 'nah', 'glk', 'co', 'hsb', 'roa-rup', 'kk', + 'als', 'ia', 'li', 'hy', 'sah', 'bcl', 'gan', 'tt', 'tk', 'sa', + 'nds-nl', 'wuu', 'os', 'fo', 'vls', 'arz', 'fiu-vro', 'nrm', 'am', 'pag', + 'rm', 'map-bms', 'dv', 'mn', 'gv', 'se', 'diq', 'fur', 'ne', 'sco', + 'bar', 'lij', 'nov', 'bh', 'mt', 'ilo', 'pi', 'mzn', 'zh-classical', 'csb', + 'ug', 'km', 'frp', 'lad', 'si', 'kw', 'ang', 'sc', 'pdc', 'haw', + 'mg', 'to', 'kv', 'my', 'szl', 'pa', 'ps', 'myv', 'ie', 'gn', + 'hif', 'ln', 'stq', 'jbo', 'wo', 'arc', 'crh', 'tpi', 'ty', 'ext', + 'cbk-zam', 'ky', 'hak', 'eml', 'zea', 'srn', 'ay', 'pap', 'so', 'kg', + 'ig', 'or', 'kab', 'lo', 'udm', 'rmy', 'ba', 'dsb', 'ce', 'sm', + 'av', 'nv', 'cu', 'ks', 'tet', 'got', 'kaa', 'bo', 'sd', 'mdf', + 'iu', 'na', 'bm', 'as', 'cdo', 'kl', 'pnt', 'chr', 'ee', 'om', + 'pih', 'ab', 'zu', 'ti', 'ts', 'ss', 'za', 'bi', 'ha', 'cr', + 'dz', 'xal', 've', 'ch', 'xh', 'tn', 'bug', 'ik', 'bxr', 'rw', + 'st', 'tw', 'chy', 'ak', 'fj', 'ny', 'ff', 'sn', 'sg', 'lbe', + 'lg', 'rn', 'ki', 'tum', 'ng', 'mhr', 'ckb', 'pnb', 'mwl', 'ace' ] for lang in self.languages_by_size: @@ -125,11 +125,15 @@ 'af': [u'Dubbelsinnig', u'Disambig'], 'als': [u'Begriffsklärung', u'Disambig'], 'an': [u'Desambig', u'Disambig'], + 'ang': [u'Disambig', u'Geodis'], 'ar': [u'Disambig', u'توضيح'], 'arc': [u'ܕ'], + 'arz': [u'توضيح'], 'ast': [u'Dixebra'], 'av': [u'Неоднозначность'], + 'ay': [u'Desambiguación'], 'az': [u'Dəqiqləşdirmə'], + 'ba': [u'Күп мәғәнәлелек'], 'bar': [u'Begriffsklärung'], 'be': [u'Неадназначнасць', u'Disambig'], 'be-x-old': [u'Неадназначнасць', u'Неадназначнасьць', u'Disambig'], @@ -141,6 +145,7 @@ u'acrònim', u'onomàstica'], 'ceb': [u'Giklaro'], 'cdo': [u'Gì-ngiê'], + 'crh': [u'Çoq manalı', u'Disambig'], 'cs': [u'Rozcestník', u'Rozcestník - 2 znaky', u'Rozcestník - Příjmení', u'Rozcestník - místopisné jméno', u'Disambig', u'Rozcestník - příjmení', u'Rozcestník - sakrální stavba', u'Rozcestník - kostel', @@ -149,6 +154,7 @@ 'cy': [u'Anamrwysedd', u'Disambig', u'Gwahaniaethu'], 'da': [u'Flertydig'], 'de': [u'Begriffsklärung', u'BKS', u'Disambig'], + 'dsb': [u'Wěcejwóznamowosć'], 'el': [u'Disambig', u'Αποσαφ', u'Αποσαφήνιση'], 'en': None, 'eo': [u'Apartigilo', u'Disambig'], @@ -158,11 +164,14 @@ 'ext': [u'Desambiguáncia'], 'fa': [u'ابهام‌زدایی',u'ابهام زدایی'], 'fi': [u'Täsmennyssivu', u'Disambig'], + 'fiu-vro': [u'Täpsüstüslehekülg'], 'fo': [u'Fleiri týdningar'], 'fr': None, 'frp': [u'Homonimos'], + 'fur': [u'Disambiguazion', u'Disambig'], 'fy': [u'Tfs', u'Neibetsjuttings'], 'ga': [u'Idirdhealú', u'Disambig'], + 'gan': [u'扤清楚', u'Disambig'], 'gl': [u'Homónimos', u'Disambig'], 'gv': [u'Reddaghey'], 'haw': [u'Huaʻōlelo puana like'], @@ -194,6 +203,7 @@ 'ln': [u'Bokokani'], 'lt': None, 'mk': [u'Појаснување', u'Disambig', u'Geodis'], + 'mn': [u'Салаа утгатай', u'Салаа', u'Disambig'], 'mo': [u'Дезамбигуйзаре', u'Disambig'], 'ms': [u'Nyahkekaburan', u'Disambig'], 'mt': [u'Diżambigwazzjoni'], @@ -213,7 +223,7 @@ 'pt': [u'Desambiguação', u'Disambig', u'Desambig'], 'qu': [u"Sut'ichana qillqa", u'Disambig', u'SJM'], 'rmy': [u'Dudalipen'], - 'ro': [u'Dezambiguizare', u'Disambig', u'Hndis', u'Dez'], + 'ro': [u'Dezambiguizare', u'Disambig', u'Hndis', u'Dez', u'Dezamb'], 'ru': [u'Disambig', u'Неоднозначность', u'неоднозначность', u'Многозначность', u'Фамилия'], 'scn': [u'Disambigua', u'Disambig', u'Sigla2', u'Sigla3'], @@ -228,6 +238,7 @@ 'sr': [u'Вишезначна одредница', u'Disambig', u'Вишезначна', u'Višeznačna odrednica'], 'srn': [u'Dp'], + 'stq': [u'Begriepskläärenge'], 'su': [u'Disambig'], 'sv': None, 'sw': [u'Maana'], @@ -236,6 +247,7 @@ 'tg': [u'Ибҳомзудоӣ', u'Disambig', u'Рафъи ибҳом', u'Disambiguation'], 'th': [u'แก้กำกวม', u'Disambig', u'คำกำกวม'], 'tl': [u'Paglilinaw', u'Disambig'], + 'to': [u'Fakaʻuhingakehe'], 'tr': [u'Anlam ayrım', u'Disambig', u'Anlam ayrımı', u'Kişi adları (anlam ayrımı)', u'Yerleşim yerleri (anlam ayrımı)', u'kısaltmalar (anlam ayrımı)'], @@ -246,6 +258,7 @@ 'vls': [u'Db', u'Dp', u'Dpintro'], 'vo': [u'Telplänov'], 'wa': [u'Omonimeye', u'Disambig'], + 'wo': [u'Bokktekki'], 'yi': [u'באדייטען'], 'zea': [u'dp', u'Deurverwiespagina'], 'zh': [u'Disambig', u'消歧义', u'消歧义页', u'消歧義'], @@ -331,30 +344,30 @@ # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementa… self.cross_allowed = [ - 'ab', 'af', 'ak', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', + 'ab', 'ace', 'af', 'ak', 'als', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', 'ba', 'bar', 'bcl', 'be-x-old', 'be', 'bg', 'bh', 'bi', 'bm', 'bo', 'bpy', 'bug', 'bxr', - 'cbk-zam', 'cdo', 'ce', 'ch', 'chr', 'chy', 'co', 'crh', 'cr', 'csb', 'cu', 'cv', 'cy', - 'diq', 'dsb', 'dz', 'ee', 'eml', 'eu', 'ext', 'fa', 'ff', 'fj', 'fo', 'frp', 'fur', - 'gan', 'ga', 'glk', 'gn', 'got', 'gu', 'gv', 'ha', 'hak', 'haw', 'hif', 'hi', 'hsb', 'ht', 'hu', 'hy', - 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'iow', 'is', 'iu', 'jbo', 'jv', + 'cbk-zam', 'cdo', 'ce', 'ch', 'chr', 'chy', 'ckb', 'co', 'crh', 'cr', 'csb', 'cu', 'cv', 'cy', + 'diq', 'dsb', 'dz', 'ee', 'el', 'eml', 'eu', 'ext', 'fa', 'ff', 'fj', 'fo', 'frp', 'fur', + 'ga', 'gan', 'glk', 'gn', 'got', 'gu', 'gv', 'ha', 'hak', 'haw', 'hif', 'hi', 'hsb', 'ht', 'hu', 'hy', + 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'iow', 'is', 'iu', 'ja', 'jbo', 'jv', 'kaa', 'kab', 'ka', 'kg', 'ki', 'kl', 'km', 'kn', 'ko', 'ks', 'ku', 'kv', 'kw', 'ky', - 'lad', 'lbe', 'lg', 'li', 'lij', 'lmo', 'ln', 'lo', 'lv', 'map-bms', 'mdf', 'mg', 'mk', 'mn', 'mt', 'myv', 'my', 'mzn', + 'lad', 'lb', 'lbe', 'lg', 'li', 'lij', 'lmo', 'ln', 'lo', 'lv', + 'map-bms', 'mdf', 'mg', 'mhr', 'mk', 'mn', 'ms', 'mt', 'mwl', 'myv', 'my', 'mzn', 'nah', 'na', 'nap', 'nds-nl', 'ne', 'new', 'ng', 'nl', 'nov', 'nrm', 'nv', 'ny', 'om', 'or', 'os', - 'pam', 'pap', 'pa', 'pdc', 'pnt', 'ps', 'qu', 'rm', 'rmy', 'rn', 'roa-rup', 'roa-tara', 'rw', + 'pam', 'pap', 'pa', 'pdc', 'pnb', 'pnt', 'ps', 'qu', 'rm', 'rmy', 'rn', 'roa-rup', 'roa-tara', 'rw', 'sah', 'sa', 'sc', 'sco', 'sd', 'se', 'sg', 'sh', 'simple', 'si', 'sk', 'sm', 'sn', 'so', 'srn', 'stq', 'st', 'su', 'sw', 'szl', 'ta', 'te', 'tet', 'tg', 'ti', 'tk', 'tn', 'to', 'tpi', 'ts', 'tt', 'tum', 'tw', 'ty', 'udm', 'ug', 'uz', 've', 'vls', 'wa', 'war', 'wo', 'wuu', - 'xal', 'xh', 'yi', 'yo', 'za', 'zh', 'zh-classic', 'zh-min-nan', 'zu', + 'xal', 'xh', 'yi', 'yo', 'za', 'zea', 'zh', 'zh-classical', 'zh-min-nan', 'zu', ] # On most Wikipedias page names must start with a capital letter, but some # languages don't use this. self.nocapitalize = ['jbo',] - # A revised sorting order worked out on http://meta.wikimedia.org/wiki/Interwiki_sorting_order self.alphabetic_revised = [ - 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', + 'af', 'ak', 'ace', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', 'frp', 'as', 'ast', 'gn', 'av', 'ay', 'az', 'id', 'ms', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'jv', 'su', 'ba', 'be', 'be-x-old', 'bh', 'bcl', 'bi', 'bar', 'bo', 'bs', 'br', 'bug', 'bg', 'bxr', @@ -368,14 +381,14 @@ 'kn', 'kr', 'pam', 'ka', 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'sw', 'kv', 'kg', 'ht', 'ku', 'kj', 'lad', 'lbe', 'lo', 'la', 'lv', 'to', 'lb', 'lt', 'lij', 'li', 'ln', 'jbo', 'lg', 'lmo', 'hu', - 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'arz', 'mzn', 'cdo', 'mdf', + 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'arz', 'mzn', 'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'fj', 'nl', 'nds-nl', 'cr', 'ne', 'new', 'ja', 'nap', 'ce', 'pih', 'no', 'nb', 'nn', 'nrm', 'nov', - 'ii', 'oc', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pi', 'pag', 'pap', + 'ii', 'oc', 'mhr', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pnb', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'nds', 'pl', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'ru', 'sah', 'se', 'sa', 'sg', 'sc', 'za', 'sco', 'stq', 'st', 'tn', 'sq', 'scn', 'si', 'simple', - 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so', 'srn', 'sr', 'sh', 'fi', + 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'fi', 'sv', 'tl', 'ta', 'kab', 'roa-tara', 'tt', 'te', 'tet', 'th', 'vi', 'ti', 'tg', 'tpi', 'tokipona', 'tp', 'chr', 'chy', 've', 'tr', 'tk', 'tw', 'udm', 'uk', 'ur', 'ug', 'vec', 'vo', 'fiu-vro', 'wa', @@ -415,6 +428,7 @@ 'pl': self.alphabetic, 'simple': self.alphabetic, 'te': ['en', 'hi', 'kn', 'ta', 'ml'], + 'ur': ['ar', 'fa', 'en'] + self.alphabetic, 'vi': self.alphabetic_revised, 'yi': ['en', 'he', 'de'] } @@ -452,6 +466,7 @@ '_default': { 'pt': [102], 'als': [104], + 'ar': [104], 'en': [12], 'es': [104], 'fi': [4], @@ -575,7 +590,7 @@ return self.known_families def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def dbName(self, code): # returns the name of the MySQL database Modified: branches/rewrite/pywikibot/families/wikiquote_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikiquote_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikiquote_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -11,13 +11,13 @@ self.name = 'wikiquote' self.languages_by_size = [ - 'en', 'it', 'de', 'pl', 'pt', 'sk', 'ru', 'bg', 'es', 'bs', - 'sl', 'tr', 'he', 'lt', 'fr', 'cs', 'zh', 'hu', 'uk', 'id', - 'fa', 'sv', 'el', 'no', 'nl', 'ja', 'fi', 'eo', 'hy', 'et', - 'nn', 'ca', 'simple', 'ka', 'ar', 'ku', 'hr', 'ko', 'ro', 'gl', + 'en', 'it', 'de', 'pl', 'pt', 'sk', 'ru', 'es', 'bg', 'bs', + 'sl', 'tr', 'he', 'lt', 'fr', 'cs', 'zh', 'fa', 'hu', 'uk', + 'id', 'sv', 'el', 'no', 'nl', 'ja', 'fi', 'eo', 'hy', 'et', + 'nn', 'ca', 'simple', 'ka', 'ar', 'ku', 'ko', 'hr', 'ro', 'gl', 'sr', 'ml', 'li', 'is', 'th', 'af', 'te', 'da', 'sq', 'vi', 'eu', 'az', 'la', 'br', 'hi', 'be', 'ast', 'uz', 'ang', 'zh-min-nan', - 'lb', 'mr', 'su', 'gu', 'ur', 'ta', 'wo', 'kn', 'ky', 'cy', + 'lb', 'mr', 'gu', 'su', 'ur', 'ta', 'wo', 'kn', 'ky', 'cy', 'am', 'co', 'kk', ] @@ -106,7 +106,7 @@ } def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def code2encodings(self, code): """ Modified: branches/rewrite/pywikibot/families/wikisource_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikisource_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikisource_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -13,9 +13,9 @@ self.languages_by_size = [ 'en', 'pt', 'fr', 'zh', 'es', 'de', 'ru', 'he', 'it', 'ar', 'fa', 'hu', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'tr', - 'nl', 'sv', 'ko', 'sl', 'sr', 'la', 'ja', 'uk', 'li', 'el', - 'ml', 'is', 'bs', 'id', 'ca', 'bn', 'yi', 'az', 'hy', 'no', - 'mk', 'da', 'vi', 'ta', 'et', 'bg', 'lt', 'kn', 'gl', 'cy', + 'nl', 'sv', 'sl', 'ko', 'sr', 'ja', 'la', 'uk', 'el', 'li', + 'ml', 'is', 'bs', 'id', 'ca', 'bn', 'az', 'yi', 'hy', 'mk', + 'no', 'da', 'vi', 'et', 'ta', 'bg', 'lt', 'kn', 'gl', 'cy', 'sk', 'zh-min-nan', 'fo', ] @@ -68,7 +68,7 @@ return '/w' def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/wikiversity_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikiversity_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wikiversity_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -19,7 +19,7 @@ self.langs[lang] = '%s.wikiversity.org' % lang def version(self,code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/families/wiktionary_family.py =================================================================== --- branches/rewrite/pywikibot/families/wiktionary_family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/families/wiktionary_family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -13,18 +13,18 @@ self.languages_by_size = [ 'fr', 'en', 'tr', 'lt', 'vi', 'ru', 'io', 'zh', 'el', 'pl', 'fi', 'no', 'hu', 'it', 'ta', 'sv', 'de', 'ko', 'lo', 'pt', - 'nl', 'ku', 'es', 'ja', 'id', 'te', 'et', 'ro', 'gl', 'bg', - 'vo', 'ar', 'uk', 'ca', 'is', 'sr', 'fa', 'af', 'li', 'scn', - 'sw', 'fy', 'th', 'br', 'oc', 'he', 'cs', 'simple', 'sl', 'hy', - 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'ur', 'hsb', 'kk', 'wa', - 'ml', 'ky', 'hr', 'wo', 'tk', 'ang', 'eo', 'kn', 'hi', 'gn', + 'nl', 'ku', 'ja', 'es', 'id', 'te', 'ro', 'et', 'gl', 'bg', + 'ca', 'ar', 'vo', 'uk', 'fa', 'is', 'sr', 'af', 'sw', 'scn', + 'fy', 'th', 'br', 'oc', 'simple', 'li', 'cs', 'he', 'sl', 'hy', + 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb', + 'kk', 'ml', 'ky', 'hr', 'wo', 'kn', 'ang', 'eo', 'hi', 'gn', 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', - 'sd', 'ug', 'ti', 'tl', 'mk', 'an', 'my', 'gu', 'ka', 'km', - 'lv', 'ts', 'cy', 'qu', 'fo', 'bs', 'am', 'rw', 'eu', 'mr', - 'su', 'chr', 'mn', 'nah', 'ie', 'yi', 'om', 'be', 'gd', 'sh', - 'iu', 'mg', 'nn', 'zu', 'bn', 'pa', 'si', 'mt', 'tpi', 'dv', - 'mi', 'jv', 'ps', 'roa-rup', 'tg', 'so', 'ik', 'ha', 'gv', 'uz', - 'ss', 'kw', 'sa', 'ay', 'na', 'ne', 'jbo', 'tn', 'sm', 'sg', + 'sd', 'ug', 'ti', 'tl', 'mk', 'ka', 'an', 'my', 'gu', 'km', + 'lv', 'cy', 'ts', 'qu', 'eu', 'fo', 'bs', 'am', 'rw', 'mr', + 'su', 'chr', 'mn', 'nah', 'om', 'ie', 'yi', 'be', 'iu', 'mg', + 'sh', 'gd', 'nn', 'bn', 'zu', 'si', 'pa', 'mt', 'dv', 'tpi', + 'mi', 'roa-rup', 'jv', 'tg', 'ps', 'ik', 'so', 'uz', 'ha', 'gv', + 'ss', 'kw', 'sa', 'ay', 'na', 'jbo', 'ne', 'tn', 'sm', 'sg', 'lb', 'ks', 'fj', 'ln', 'za', 'dz', 'als', ] @@ -107,7 +107,7 @@ self.interwiki_attop = ['pl'] def version(self, code): - return '1.16alpha' + return '1.16alpha-wmf' def shared_image_repository(self, code): return ('commons', 'commons') Modified: branches/rewrite/pywikibot/family.py =================================================================== --- branches/rewrite/pywikibot/family.py 2009-08-26 03:16:54 UTC (rev 7175) +++ branches/rewrite/pywikibot/family.py 2009-08-26 17:50:13 UTC (rev 7176) @@ -23,7 +23,7 @@ self.name = None # Updated from http://meta.wikimedia.org/wiki/Interwiki_sorting_order self.alphabetic = [ - 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', + 'af', 'ak', 'ace', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', 'frp', 'as', 'ast', 'gn', 'av', 'ay', 'az', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'ba', 'be', 'be-x-old', 'bh', 'bcl', 'bi', 'bar', 'bo', 'bs', 'br', 'bg', 'bxr', 'ca', 'cv', 'ceb', 'cs', @@ -37,14 +37,14 @@ 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'sw', 'kv', 'kg', 'ht', 'ku', 'kj', 'lad', 'lbe', 'lo', 'la', 'lv', 'lb', 'lt', 'lij', 'li', 'ln', 'jbo', 'lg', 'lmo', 'hu', 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', - 'arz', 'mzn', 'ms', 'cdo', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', + 'arz', 'mzn', 'ms', 'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'na', 'fj', 'nl', 'nds-nl', 'cr', 'ne', 'new', 'ja', 'nap', 'ce', - 'pih', 'no', 'nb', 'nn', 'nrm', 'nov', 'ii', 'oc', 'or', 'om', 'ng', - 'hz', 'uz', 'pa', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'tpi', + 'pih', 'no', 'nb', 'nn', 'nrm', 'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng', + 'hz', 'uz', 'pa', 'pnb', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'tpi', 'nds', 'pl', 'tokipona', 'tp', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'ru', 'sah', 'se', 'sm', 'sa', 'sg', 'sc', 'za', 'sco', 'stq', 'st', 'tn', 'sq', 'scn', 'si', - 'simple', 'sd', 'ss', 'sk', 'cu', 'sl', 'szl', 'so', 'srn', 'sr', + 'simple', 'sd', 'ss', 'sk', 'cu', 'sl', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'su', 'fi', 'sv', 'tl', 'ta', 'kab', 'roa-tara', 'tt', 'te', 'tet', 'th', 'ti', 'tg', 'to', 'chr', 'chy', 've', 'tr', 'tk', 'tw', 'udm', 'bug', 'uk', 'ur', 'ug', 'vec', 'vi', 'vo', 'fiu-vro', @@ -68,6 +68,7 @@ 'he': u'[a-zא-ת]*', 'it': u'[a-zàèéìòù]*', 'kk': u'[a-zäçéğıïñöşüýа-яёәғіқңөұүһʺʹ]*', + 'ksh': u'[äöüėëĳßəğåůæœça-z]*', 'nl': u'[a-zäöüïëéèéàç]*', 'pt': u'[a-záâàãéêíóôõúüç]*', 'ru': u'[a-zа-я]*', @@ -547,9 +548,9 @@ self.language_groups = { # languages using the arabic script (incomplete) 'arab' : [ - 'ar', 'arz', 'ps', 'sd', 'ur', + 'ar', 'arz', 'ps', 'sd', 'ur', 'ckb', # languages using multiple scripts, including arabic - 'kk', 'ku', 'tt', 'ug' + 'kk', 'ku', 'tt', 'ug', 'pnb' ], # languages that use chinese symbols 'chinese': [ @@ -565,9 +566,14 @@ # languages using multiple scripts, including cyrillic 'ha', 'kk', 'sh', 'sr', 'tt' ], + # languages that use a greek script + 'grec': [ + 'el', 'grc', 'pnt' + # languages using multiple scripts, including greek + ], # languages that use the latin alphabet 'latin': [ - 'aa', 'af', 'ak', 'als', 'an', 'ang', 'ast', 'ay', 'bar', 'bat-smg', + 'aa', 'ace', 'af', 'ak', 'als', 'an', 'ang', 'ast', 'ay', 'bar', 'bat-smg', 'bcl', 'bi', 'bm', 'br', 'bs', 'ca', 'cbk-zam', 'cdo', 'ceb', 'ch', 'cho', 'chy', 'co', 'crh', 'cs', 'csb', 'cy', 'da', 'de', 'diq', 'dsb', 'ee', 'eml', 'en', 'eo', 'es', 'et', 'eu', 'ext', 'ff', 'fi', 'fiu-vro', @@ -576,7 +582,7 @@ 'ie', 'ig', 'ik', 'ilo', 'io', 'is', 'it', 'jbo', 'jv', 'kaa', 'kab', 'kg', 'ki', 'kj', 'kl', 'kr', 'ksh', 'kw', 'la', 'lad', 'lb', 'lg', 'li', 'lij', 'lmo', 'ln', 'lt', 'lv', 'map-bms', 'mg', 'mh', 'mi', 'ms', - 'mt', 'mus', 'na', 'nah', 'nap', 'nds', 'nds-nl', 'ng', 'nl', 'nn', + 'mt', 'mus', 'mwl', 'na', 'nah', 'nap', 'nds', 'nds-nl', 'ng', 'nl', 'nn', 'no', 'nov', 'nrm', 'nv', 'ny', 'oc', 'om', 'pag', 'pam', 'pap', 'pdc', 'pih', 'pl', 'pms', 'pt', 'qu', 'rm', 'rn', 'ro', 'roa-rup', 'roa-tara', 'rw', 'sc', 'scn', 'sco', 'se', 'sg', 'simple', 'sk', 'sl', 'sm', 'sn', @@ -585,12 +591,12 @@ 'vec', 'vi', 'vls', 'vo', 'wa', 'war', 'wo', 'xh', 'yo', 'zea', 'zh-min-nan', 'zu', # languages using multiple scripts, including latin - 'az', 'chr', 'ha', 'iu', 'kk', 'ku', 'rmy', 'sh', 'sr', - 'tt', 'ug', 'za' + 'az', 'chr', 'ckb', 'ha', 'iu', 'kk', 'ku', 'rmy', 'sh', 'sr', 'tt', + 'ug', 'za' ], # Scandinavian languages 'scand': [ - 'da', 'fo', 'is', 'no', 'sv' + 'da', 'fo', 'is', 'nb', 'nn', 'no', 'sv' ], } @@ -784,7 +790,7 @@ 'pl': [u'patrz', u'przekieruj', u'tam'], 'pms': [u'rinvia', u'rinvio'], 'pt-br': [u'redirecionamento'], - 'qu': [u'redirección'], + 'qu': [u'pusapuna', u'redirección'], 'rmy': [u'redirecteaza'], 'ro': [u'redirecteaza'], 'ru': [u'перенаправление', u'перенапр'], Property changes on: branches/rewrite/pywikibot/family.py ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/pywikipedia/family.py:6057-6972,6974-7067 + /trunk/pywikipedia/family.py:6057-6972,6974-7067,7070-7175

1 0

SVN: [7175] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 25 Aug '09

25 Aug '09

Revision: 7175 Author: alexsh Date: 2009-08-26 03:16:54 +0000 (Wed, 26 Aug 2009) Log Message: ----------- UpNS:ar.ws, bn.wp(+mainFamily) Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/families/wikisource_family.py trunk/pywikipedia/family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-26 03:15:47 UTC (rev 7174) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-26 03:16:54 UTC (rev 7175) @@ -67,7 +67,7 @@ 'be': u'Вікіпедыя', 'be-x-old': u'Вікіпэдыя', 'bg': u'Уикипедия', - 'bn': u'উইকিপেডিয়া', + 'bn': u'উইকিপিডিয়া', 'bpy': u'উইকিপিডিয়া', 'ca': u'Viquipèdia', 'ce': u'Википедийа', @@ -169,7 +169,7 @@ 'be-x-old': u'Абмеркаваньне Вікіпэдыя', 'bg': u'Уикипедия беседа', 'bm': u'Discussion Wikipedia', - 'bn': u'উইকিপেডিয়া আলোচনা', + 'bn': u'উইকিপিডিয়া আলোচনা', 'bpy': u'উইকিপিডিয়া য়্যারী', 'br': [u'Kaozeadenn Wikipedia', u'Discussion Wikipedia'], 'bs': u'Razgovor s Wikipediom', Modified: trunk/pywikipedia/families/wikisource_family.py =================================================================== --- trunk/pywikipedia/families/wikisource_family.py 2009-08-26 03:15:47 UTC (rev 7174) +++ trunk/pywikipedia/families/wikisource_family.py 2009-08-26 03:16:54 UTC (rev 7175) @@ -130,6 +130,7 @@ 'yi': [u'װיקיביבליאָטעק רעדן', u'וויקיביבליאטעק רעדן'], } self.namespaces[100] = { + 'ar': u'بوابة', 'bg': u'Автор', 'bn': u'লেখক', 'cs': u'Autor', @@ -148,6 +149,7 @@ 'tr': u'Kişi', } self.namespaces[101] = { + 'ar': u'نقاش البوابة', 'bg': u'Автор беседа', 'bn': u'লেখক আলাপ', 'cs': u'Autor diskuse', @@ -253,6 +255,7 @@ } self.namespaces[106] = { + 'ar': u'فهرس', 'en': u'Index', 'he': u'ביאור', 'hu': u'Index', @@ -267,6 +270,7 @@ } self.namespaces[107] = { + 'ar': u'نقاش الفهرس', 'en': u'Index talk', 'he': u'שיחת ביאור', 'hu': u'Index vita', Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-26 03:15:47 UTC (rev 7174) +++ trunk/pywikipedia/family.py 2009-08-26 03:16:54 UTC (rev 7175) @@ -895,6 +895,7 @@ 4: { '_default': u'Project', 'arc': u'ܘܝܩܝܦܕܝܐ', + 'bn': u'উইকিপিডিয়া', 'cs': u'Wikizdroje', 'ko': u'위키문헌', 'uk': u'Вікіцитати', @@ -902,6 +903,7 @@ 5: { '_default': u'Project talk', 'arc': u'ܘܝܩܝܦܕܝܐ talk', + 'bn': u'উইকিপিডিয়া আলোচনা', 'cs': u'Wikizdroje diskuse', 'gan': u'Wikipedia talk', 'ko': u'위키문헌토론',

1 0

SVN: [7174] trunk/pywikipedia/flickrripper.py
by alexsh＠svn.wikimedia.org 25 Aug '09

25 Aug '09

Revision: 7174 Author: alexsh Date: 2009-08-26 03:15:47 +0000 (Wed, 26 Aug 2009) Log Message: ----------- syntax typo Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-25 19:09:02 UTC (rev 7173) +++ trunk/pywikipedia/flickrripper.py 2009-08-26 03:15:47 UTC (rev 7174) @@ -67,7 +67,8 @@ ''' result = [] for tag in photoInfo.find('photo').find('tags').findall('tag'): - result.append(tag.text.lower()) + result.append(tag.text.lower()) + return result def getFlinfoDescription(photo_id=0): @@ -88,9 +89,9 @@ username = photoInfo.find('photo').find('owner').attrib['username'] title = photoInfo.find('photo').find('title').text if title: - title = cleanUpTitle(title) + title = cleanUpTitle(title) else: - title = u'' + title = u'' return u'Flickr - %s - %s.jpg' % (username, title) @@ -123,7 +124,7 @@ url = '' # The assumption is that the largest image is last for size in photoSizes.find('sizes').findall('size'): - url = size.attrib['source'] + url = size.attrib['source'] return url def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): @@ -181,7 +182,7 @@ print photo_id (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) if (isAllowedLicense(photoInfo=photoInfo) or override): - # Tags not needed atm + # Tags not needed atm #tags=getTags(photoInfo=photoInfo) flinfoDescription = getFlinfoDescription(photo_id=photo_id) @@ -190,10 +191,10 @@ #print filename photoUrl = getPhotoUrl(photoSizes=photoSizes) #print photoUrl - photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) - #wikipedia.output(photoDescription) - (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() - #wikipedia.output(newPhotoDescription) + photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) + #wikipedia.output(photoDescription) + (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() + #wikipedia.output(newPhotoDescription) #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): # I should probably check if the hash is the same and if not upload it under a different name #wikipedia.output(u'File:' + filename + u' already exists!') @@ -202,8 +203,8 @@ #Would be nice to check before I upload if the file is already at Commons #Not that important for this program, but maybe for derived programs if not skip: - bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) - bot.upload_image(debug=False) + bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) + bot.upload_image(debug=False) return 0 class Tkdialog:

1 0

SVN: [7173] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 25 Aug '09

25 Aug '09

Revision: 7173 Author: nicdumz Date: 2009-08-25 19:09:02 +0000 (Tue, 25 Aug 2009) Log Message: ----------- change MarcoBot Name in summary too. Modified Paths: -------------- trunk/pywikipedia/reflinks.py Modified: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py 2009-08-25 18:42:11 UTC (rev 7172) +++ trunk/pywikipedia/reflinks.py 2009-08-25 19:09:02 UTC (rev 7173) @@ -57,7 +57,7 @@ 'hu':u'Robot: Forráshivatkozások kibővítése a hivatkozott oldal címével', 'ko':u'봇: url만 있는 주석을 보강, (영문)[[:en:User:DumZiBoT/refLinks]] 참조', 'es':u'Formateando las referencias que no tuvieran títulos (FAQ : [[:en:User:DumZiBoT/refLinks]] )', - 'it':u'Bot: Sistemo note con collegamenti esterni senza titolo ([[Utente:Màrço27Bot/refLinks.py|documentazione]])', + 'it':u'Bot: Sistemo note con collegamenti esterni senza titolo ([[Utente:Marco27Bot/refLinks.py|documentazione]])', 'en':u'Bot: Converting bare references, using ref names to avoid duplicates, see [[User:DumZiBoT/refLinks|FAQ]]', }

1 0

SVN: [7172] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 25 Aug '09

25 Aug '09

Revision: 7172 Author: nicdumz Date: 2009-08-25 18:42:11 +0000 (Tue, 25 Aug 2009) Log Message: ----------- reflinks: MarcoBot has been renamed Modified Paths: -------------- trunk/pywikipedia/reflinks.py Modified: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py 2009-08-24 17:06:26 UTC (rev 7171) +++ trunk/pywikipedia/reflinks.py 2009-08-25 18:42:11 UTC (rev 7172) @@ -45,7 +45,7 @@ stopPage = {'fr':u'Utilisateur:DumZiBoT/EditezCettePagePourMeStopper', 'de':u'Benutzer:DumZiBoT/EditThisPageToStopMe', - 'it':u'Utente:Màrço27Bot/EditThisPageToStopMe', + 'it':u'Utente:Marco27Bot/EditThisPageToStopMe', 'ko':u'사용자:GrassnBreadRefBot/EditThisPageToStopMe1', 'hu':'User:Damibot/EditThisPageToStopMe', 'en':u'User:DumZiBoT/EditThisPageToStopMe',

1 0

SVN: [7171] trunk/pywikipedia/flickrripper.py
by alexsh＠svn.wikimedia.org 24 Aug '09

24 Aug '09

Revision: 7171 Author: alexsh Date: 2009-08-24 17:06:26 +0000 (Mon, 24 Aug 2009) Log Message: ----------- typo Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-24 17:04:31 UTC (rev 7170) +++ trunk/pywikipedia/flickrripper.py 2009-08-24 17:06:26 UTC (rev 7171) @@ -55,11 +55,11 @@ ''' license = photoInfo.find('photo').attrib['license'] if (license=='4' or license=='5'): - #Is cc-by or cc-by-sa - return True + #Is cc-by or cc-by-sa + return True else: - #We don't accept other licenses - return False + #We don't accept other licenses + return False def getTags(photoInfo=None): '''

1 0

SVN: [7170] trunk/pywikipedia/flickrripper.py
by alexsh＠svn.wikimedia.org 24 Aug '09

24 Aug '09

Revision: 7170 Author: alexsh Date: 2009-08-24 17:04:31 +0000 (Mon, 24 Aug 2009) Log Message: ----------- typo Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2009-08-23 16:13:23 UTC (rev 7169) +++ trunk/pywikipedia/flickrripper.py 2009-08-24 17:04:31 UTC (rev 7170) @@ -55,11 +55,11 @@ ''' license = photoInfo.find('photo').attrib['license'] if (license=='4' or license=='5'): - #Is cc-by or cc-by-sa - return True + #Is cc-by or cc-by-sa + return True else: - #We don't accept other licenses - return False + #We don't accept other licenses + return False def getTags(photoInfo=None): ''' @@ -67,7 +67,7 @@ ''' result = [] for tag in photoInfo.find('photo').find('tags').findall('tag'): - result.append(tag.text.lower()) + result.append(tag.text.lower()) return result def getFlinfoDescription(photo_id=0): @@ -88,9 +88,9 @@ username = photoInfo.find('photo').find('owner').attrib['username'] title = photoInfo.find('photo').find('title').text if title: - title = cleanUpTitle(title) + title = cleanUpTitle(title) else: - title = u'' + title = u'' return u'Flickr - %s - %s.jpg' % (username, title) @@ -123,7 +123,7 @@ url = '' # The assumption is that the largest image is last for size in photoSizes.find('sizes').findall('size'): - url = size.attrib['source'] + url = size.attrib['source'] return url def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): @@ -180,28 +180,28 @@ if(photo_id): print photo_id (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) - if (isAllowedLicense(photoInfo=photoInfo) or override): + if (isAllowedLicense(photoInfo=photoInfo) or override): # Tags not needed atm - #tags=getTags(photoInfo=photoInfo) + #tags=getTags(photoInfo=photoInfo) - flinfoDescription = getFlinfoDescription(photo_id=photo_id) + flinfoDescription = getFlinfoDescription(photo_id=photo_id) - filename = getFilename(photoInfo=photoInfo) - #print filename - photoUrl = getPhotoUrl(photoSizes=photoSizes) - #print photoUrl + filename = getFilename(photoInfo=photoInfo) + #print filename + photoUrl = getPhotoUrl(photoSizes=photoSizes) + #print photoUrl photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) #wikipedia.output(photoDescription) (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() #wikipedia.output(newPhotoDescription) - #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): - # I should probably check if the hash is the same and if not upload it under a different name - #wikipedia.output(u'File:' + filename + u' already exists!') - #else: - #Do the actual upload - #Would be nice to check before I upload if the file is already at Commons - #Not that important for this program, but maybe for derived programs - if not skip: + #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): + # I should probably check if the hash is the same and if not upload it under a different name + #wikipedia.output(u'File:' + filename + u' already exists!') + #else: + #Do the actual upload + #Would be nice to check before I upload if the file is already at Commons + #Not that important for this program, but maybe for derived programs + if not skip: bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) bot.upload_image(debug=False) return 0 @@ -326,12 +326,12 @@ flickrreview = False # Set the Flickr reviewer - if(config.flickr['reviewer']): + if config.flickr['reviewer']: reviewer = config.flickr['reviewer'] - elif(config.sysopnames['commons']['commons']): + elif config.sysopnames['commons']['commons']: reviewer = config.sysopnames['commons']['commons'] - elif(config.usernames['Commons']['Commons']): - reviewer = config.usernames['Commons']['Commons'] + elif config.usernames['commons']['commons']: + reviewer = config.usernames['commons']['commons'] else: reviewer = u''

1 0

SVN: [7169] trunk/pywikipedia/flickrripper.py
by multichill＠svn.wikimedia.org 23 Aug '09

23 Aug '09

Revision: 7169 Author: multichill Date: 2009-08-23 16:13:23 +0000 (Sun, 23 Aug 2009) Log Message: ----------- First version of flickrripper. It works, but I still have to do a lot to get a nice program. Added Paths: ----------- trunk/pywikipedia/flickrripper.py Added: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py (rev 0) +++ trunk/pywikipedia/flickrripper.py 2009-08-23 16:13:23 UTC (rev 7169) @@ -0,0 +1,389 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Tool to copy a flickr stream to Commons + +# Get a set to work on (start with just a username). +# * Make it possible to delimit the set (from/to) +#For each image +#*Check the license +#*Check if it isn't already on Commons +#*Build suggested filename +#**Check for name collision and maybe alter it +#*Pull description from Flinfo +#*Show image and description to user +#**Add a nice hotcat lookalike for the adding of categories +#**Filter the categories +#*Upload the image + +Todo: +*Check if the image is already uploaded (SHA hash) +*Check and prevent filename collisions +**Initial suggestion +**User input +*Filter the categories + +''' +# +# (C) Multichill, 2009 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id: upload_to_commons.py 69 2009-08-23 11:44:26Z multichill $' + +import sys, urllib, re, StringIO +import wikipedia, config, query, imagerecat, upload + +import flickrapi +import xml.etree.ElementTree +from Tkinter import * +from PIL import Image, ImageTk + +def getPhoto(flickr=None, photo_id=''): + ''' + Get the photo info and the photo sizes so we can use these later on + ''' + photoInfo = flickr.photos_getInfo(photo_id=photo_id) + #xml.etree.ElementTree.dump(photoInfo) + photoSizes = flickr.photos_getSizes(photo_id=photo_id) + #xml.etree.ElementTree.dump(photoSizes) + return (photoInfo, photoSizes) + +def isAllowedLicense(photoInfo=None): + ''' + Check if the image contains the right license + ''' + license = photoInfo.find('photo').attrib['license'] + if (license=='4' or license=='5'): + #Is cc-by or cc-by-sa + return True + else: + #We don't accept other licenses + return False + +def getTags(photoInfo=None): + ''' + Get all the tags on a photo + ''' + result = [] + for tag in photoInfo.find('photo').find('tags').findall('tag'): + result.append(tag.text.lower()) + return result + +def getFlinfoDescription(photo_id=0): + ''' + Get the description from http://wikipedia.ramselehof.de/flinfo.php + ''' + parameters = urllib.urlencode({'id' : photo_id, 'raw' : 'on'}) + + #print 'Flinfo gaat nu aan de slag' + rawDescription = urllib.urlopen("http://wikipedia.ramselehof.de/flinfo.php?%s" % parameters).read() + #print rawDescription.decode('utf-8') + return rawDescription.decode('utf-8') + +def getFilename(photoInfo=None): + ''' + Build a good filename for the upload based on the username and the title + ''' + username = photoInfo.find('photo').find('owner').attrib['username'] + title = photoInfo.find('photo').find('title').text + if title: + title = cleanUpTitle(title) + else: + title = u'' + + return u'Flickr - %s - %s.jpg' % (username, title) + +def cleanUpTitle(title): + title = title.strip() + + title = re.sub("[<{\\[]", "(", title) + title = re.sub("[>}\\]]", ")", title) + title = re.sub("[ _]?\$!\$", "", title) + title = re.sub(",:[ _]", ", ", title) + title = re.sub("[;:][ _]", ", ", title) + title = re.sub("[\t\n ]+", " ", title) + title = re.sub("[\r\n ]+", " ", title) + title = re.sub("[\n]+", "", title) + title = re.sub("[?!]([.\"]|$)", "\\1", title) + title = re.sub("[&#%?!]", "^", title) + title = re.sub("[;]", ",", title) + title = re.sub("[/+\\\\:]", "-", title) + title = re.sub("--+", "-", title) + title = re.sub(",,+", ",", title) + title = re.sub("[-,^]([.]|$)", "\\1", title) + title = title.replace(" ", "_") + + return title + +def getPhotoUrl(photoSizes=None): + ''' + Get the url of the jpg file with the highest resolution + ''' + url = '' + # The assumption is that the largest image is last + for size in photoSizes.find('sizes').findall('size'): + url = size.attrib['source'] + return url + +def buildDescription(flinfoDescription=u'', flickrreview=False, reviewer=u'', override=u''): + ''' + Build the final description for the image. The description is based on the info from flickrinfo and improved. + ''' + description = flinfoDescription + + if(override): + description = description.replace(u'{{cc-by-sa-2.0}}\n', u'') + description = description.replace(u'{{cc-by-2.0}}\n', u'') + description = description.replace(u'{{flickrreview}}\n', u'') + description = description.replace(u'{{copyvio|Flickr, licensed as "All Rights Reserved" which is not a free license --~~~~}}\n', u'') + description = description.replace(u'=={{int:license}}==', u'=={{int:license}}==\n' + override) + elif(flickrreview): + if(reviewer): + description = description.replace(u'{{flickrreview}}', u'{{flickrreview|' + reviewer + '|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}') + description = description.replace(u'\r\n', u'\n') + return description + +def getPhotos(flickr=None, user_id=u'', group_id=u'', photoset_id=u'', tags=u''): + result = [] + # http://www.flickr.com/services/api/flickr.groups.pools.getPhotos.html + if(group_id): + #First get the total number of photo's in the group + photos = flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.groups_pools_getPhotos(group_id=group_id, user_id=user_id, tags=tags, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.photosets.getPhotos.html + elif(photoset_id): + photos = flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.photosets_getPhotos(photoset_id=photoset_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + + # http://www.flickr.com/services/api/flickr.people.getPublicPhotos.html + elif(user_id): + photos = flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page='1') + pages = photos.find('photos').attrib['pages'] + + for i in range(1, int(pages)): + for photo in flickr.people_getPublicPhotos(user_id=user_id, per_page='100', page=i).find('photos').getchildren(): + yield photo.attrib['id'] + return + + +def processPhoto(flickr=None, photo_id=u'', flickrreview=False, reviewer=u'', override=u''): + if(photo_id): + print photo_id + (photoInfo, photoSizes) = getPhoto(flickr=flickr, photo_id=photo_id) + if (isAllowedLicense(photoInfo=photoInfo) or override): + # Tags not needed atm + #tags=getTags(photoInfo=photoInfo) + + flinfoDescription = getFlinfoDescription(photo_id=photo_id) + + filename = getFilename(photoInfo=photoInfo) + #print filename + photoUrl = getPhotoUrl(photoSizes=photoSizes) + #print photoUrl + photoDescription = buildDescription(flinfoDescription=flinfoDescription, flickrreview=flickrreview, reviewer=reviewer, override=override) + #wikipedia.output(photoDescription) + (newPhotoDescription, newFilename, skip)=Tkdialog(photoDescription, photoUrl, filename).run() + #wikipedia.output(newPhotoDescription) + #if (wikipedia.Page(title=u'File:'+ filename, site=wikipedia.getSite()).exists()): + # I should probably check if the hash is the same and if not upload it under a different name + #wikipedia.output(u'File:' + filename + u' already exists!') + #else: + #Do the actual upload + #Would be nice to check before I upload if the file is already at Commons + #Not that important for this program, but maybe for derived programs + if not skip: + bot = upload.UploadRobot(url=photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) + bot.upload_image(debug=False) + return 0 + +class Tkdialog: + def __init__(self, photoDescription, photoUrl, filename): + self.root=Tk() + #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) + self.root.geometry("1600x1000+10-10") + + self.root.title(filename) + self.photoDescription = photoDescription + self.filename = filename + self.photoUrl = photoUrl + self.skip=False + self.exit=False + + ## Init of the widgets + # The image + self.image=self.getImage(self.photoUrl, 800, 600) + self.imagePanel=Label(self.root, image=self.image) + + self.imagePanel.image = self.image + + # The filename + self.filenameLabel=Label(self.root,text=u"Suggested filename") + self.filenameField=Entry(self.root, width=100) + self.filenameField.insert(END, filename) + + # The description + self.descriptionLabel=Label(self.root,text=u"Suggested description") + self.descriptionScrollbar=Scrollbar(self.root, orient=VERTICAL) + self.descriptionField=Text(self.root) + self.descriptionField.insert(END, photoDescription) + self.descriptionField.config(state=NORMAL, height=12, width=100, padx=0, pady=0, wrap=WORD, yscrollcommand=self.descriptionScrollbar.set) + self.descriptionScrollbar.config(command=self.descriptionField.yview) + + # The buttons + self.okButton=Button(self.root, text="OK", command=self.okFile) + self.skipButton=Button(self.root, text="Skip", command=self.skipFile) + + ## Start grid + # The image + self.imagePanel.grid(row=0, column=0, rowspan=11, columnspan=4) + + # The filename + self.filenameLabel.grid(row=11, column=0) + self.filenameField.grid(row=11, column=1, columnspan=3) + + # The description + self.descriptionLabel.grid(row=12, column=0) + self.descriptionField.grid(row=12, column=1, columnspan=3) + self.descriptionScrollbar.grid(row=12, column=5) + + # The buttons + self.okButton.grid(row=13, column=1, rowspan=2) + self.skipButton.grid(row=13, column=2, rowspan=2) + + def getImage(self, url, width, height): + image=urllib.urlopen(url).read() + output = StringIO.StringIO(image) + image2 = Image.open(output) + image2.thumbnail((width, height)) + imageTk = ImageTk.PhotoImage(image2) + return imageTk + + def okFile(self): + ''' + The user pressed the OK button. + ''' + self.filename=self.filenameField.get() + self.photoDescription=self.descriptionField.get(0.0, END) + self.root.destroy() + + def skipFile(self): + ''' + The user pressed the Skip button. + ''' + self.skip=True + self.root.destroy() + + def run(self): + ''' + Activate the dialog and return the new name and if the image is skipped. + ''' + self.root.mainloop() + return (self.photoDescription, self.filename, self.skip) + + +def usage(): + wikipedia.output(u"Flickrripper is a tool to transfer flickr photos to Wikimedia Commons") + wikipedia.output(u"-group_id:<group_id>\n") + wikipedia.output(u"-photoset_id:<photoset_id>\n") + wikipedia.output(u"-user_id:<user_id>\n") + wikipedia.output(u"-tags:<tag>\n") + return + +def main(): + site = wikipedia.getSite(u'commons', u'commons') + wikipedia.setSite(site) + #imagerecat.initLists() + + #Get the api key + if(config.flickr['api_key']): + flickr = flickrapi.FlickrAPI(config.flickr['api_key']) + else: + wikipedia.output('Flickr api key not found! Get yourself an api key') + wikipedia.output('Any flickr user can get a key at http://www.flickr.com/services/api/keys/apply/') + return + + group_id = u'' + photoset_id = u'' + user_id = u'' + tags = u'' + totalPhotos = 0 + uploadedPhotos = 0 + + # Do we mark the images as reviewed right away? + if(config.flickr['review']): + flickrreview = config.flickr['review'] + else: + flickrreview = False + + # Set the Flickr reviewer + if(config.flickr['reviewer']): + reviewer = config.flickr['reviewer'] + elif(config.sysopnames['commons']['commons']): + reviewer = config.sysopnames['commons']['commons'] + elif(config.usernames['Commons']['Commons']): + reviewer = config.usernames['Commons']['Commons'] + else: + reviewer = u'' + + override = u'' + + for arg in wikipedia.handleArgs(): + if arg.startswith('-group_id'): + if len(arg) == 9: + group_id = wikipedia.input(u'What is the group_id of the pool?') + else: + group_id = arg[10:] + elif arg.startswith('-photoset_id'): + if len(arg) == 12: + photoset_id = wikipedia.input(u'What is the photoset_id)?') + else: + photoset_id = arg[13:] + elif arg.startswith('-user_id'): + if len(arg) == 8: + user_id = wikipedia.input(u'What is the user_id of the flickr user?') + else: + user_id = arg[9:] + elif arg.startswith('-tags'): + if len(arg) == 5: + tags = wikipedia.input(u'What is the tag you want to filter out (currently only one supported)?') + else: + tags = arg[6:] + elif arg == '-flickrreview': + flickrreview = True + elif arg.startswith('-reviewer'): + if len(arg) == 9: + reviewer = wikipedia.input(u'Who is the reviewer?') + else: + reviewer = arg[10:] + elif arg.startswith('-override'): + if len(arg) == 9: + override = wikipedia.input(u'What is the override text?') + else: + override = arg[10:] + + if(user_id or group_id or photoset_id): + for photo_id in getPhotos(flickr=flickr, user_id=user_id, group_id=group_id, photoset_id=photoset_id, tags=tags): + uploadedPhotos = uploadedPhotos + processPhoto(flickr=flickr, photo_id=photo_id, flickrreview=flickrreview, reviewer=reviewer, override=override) + totalPhotos = totalPhotos + 1 + else: + usage() + + wikipedia.output(u'Finished running') + wikipedia.output(u'Total photos: ' + str(totalPhotos)) + wikipedia.output(u'Uploaded photos: ' + str(uploadedPhotos)) + +if __name__ == "__main__": + try: + main() + finally: + wikipedia.stopme()

1 0

SVN: [7168] trunk/pywikipedia/imageuncat.py
by multichill＠svn.wikimedia.org 23 Aug '09

23 Aug '09

Revision: 7168 Author: multichill Date: 2009-08-23 11:33:19 +0000 (Sun, 23 Aug 2009) Log Message: ----------- Add 1 template and remove 1 template Modified Paths: -------------- trunk/pywikipedia/imageuncat.py Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2009-08-21 18:05:58 UTC (rev 7167) +++ trunk/pywikipedia/imageuncat.py 2009-08-23 11:33:19 UTC (rev 7168) @@ -85,7 +85,6 @@ u'Ay', u'Az', u'Ba', - u'Bad name', u'BadJPEG', u'Bar', u'BArch-License', @@ -453,6 +452,7 @@ u'Iu', u'Ja', u'Ja2', + u'James K. Lindsey permission', u'Jbo', u'Jean-PolGRANDMONTCredit', u'JewishEncyclopedia',

1 0

SVN: [7167] branches/rewrite/pywikibot/pagegenerators.py
by russblau＠svn.wikimedia.org 21 Aug '09

21 Aug '09

Revision: 7167 Author: russblau Date: 2009-08-21 18:05:58 +0000 (Fri, 21 Aug 2009) Log Message: ----------- Add command-line support for 'step' and 'limit' parameters of page generators. Modified Paths: -------------- branches/rewrite/pywikibot/pagegenerators.py Modified: branches/rewrite/pywikibot/pagegenerators.py =================================================================== --- branches/rewrite/pywikibot/pagegenerators.py 2009-08-20 22:37:32 UTC (rev 7166) +++ branches/rewrite/pywikibot/pagegenerators.py 2009-08-21 18:05:58 UTC (rev 7167) @@ -91,6 +91,9 @@ pages on several wiki sites, this is not well tested, so check your edits! +-limit:n When used with any other argument that specifies a set + of pages, work on no more than n pages in total + -links Work on all pages that are linked from a certain page. Argument can also be given as "-links:linkingpagetitle". @@ -118,6 +121,10 @@ -regex Obsolete, use -titleregex +-step:n When used with any other argument that specifies a set + of pages, only retrieve n pages at a time from the wiki + server + -titleregex Work on titles that match the given regular expression. -transcludes Work on all pages that use a certain template. @@ -155,6 +162,8 @@ def __init__(self): self.gens = [] self.namespaces = [] + self.step = None + self.limit = None def getCombinedGenerator(self): """Return the combination of all accumulated generators. @@ -162,14 +171,21 @@ Only call this after all arguments have been parsed. """ - if self.namespaces: - namespaces = [int(n) for n in self.namespaces] - for i in xrange(len(self.gens)): - if isinstance(self.gens[i], pywikibot.data.api.QueryGenerator): + namespaces = [int(n) for n in self.namespaces] + for i in xrange(len(self.gens)): + if isinstance(self.gens[i], pywikibot.data.api.QueryGenerator): + if self.namespaces: self.gens[i].set_namespace(namespaces) - else: + if self.step: + self.gens[i].set_query_increment(self.step) + if self.limit: + self.gens[i].set_maximum_items(self.limit) + else: + if self.namespaces: self.gens[i] = NamespaceFilterPageGenerator( self.gens[i], namespaces) + if self.limit: + self.gens[i] = itertools.islice(self.gens[i], self.limit) if len(self.gens) == 0: return None elif len(self.gens) == 1: @@ -282,6 +298,18 @@ else: self.namespaces.extend(arg[len('-ns:'):].split(",")) return True + elif arg.startswith('-step'): + if len(arg) == len('-step'): + self.step = int(pywikibot.input("What is the step value?")) + else: + self.step = int(arg[len('-step:'):]) + return True + elif arg.startswith('-limit'): + if len(arg) == len('-limit'): + self.limit = int(pywikibot.input("What is the limit value?")) + else: + self.limit = int(arg[len('-limit:'):]) + return True elif arg.startswith('-catr'): gen = self.getCategoryGen(arg, len("-catr"), recurse = True) elif arg.startswith('-category'):

1 0

SVN: [7166] trunk/pywikipedia/families/omegawiki_family.py
by siebrand＠svn.wikimedia.org 20 Aug '09

20 Aug '09

Revision: 7166 Author: siebrand Date: 2009-08-20 22:37:32 +0000 (Thu, 20 Aug 2009) Log Message: ----------- Update omegawiki_family.py Modified Paths: -------------- trunk/pywikipedia/families/omegawiki_family.py Modified: trunk/pywikipedia/families/omegawiki_family.py =================================================================== --- trunk/pywikipedia/families/omegawiki_family.py 2009-08-20 20:53:10 UTC (rev 7165) +++ trunk/pywikipedia/families/omegawiki_family.py 2009-08-20 22:37:32 UTC (rev 7166) @@ -14,10 +14,10 @@ self.langs['omegawiki'] = 'www.omegawiki.org' self.namespaces[4] = { - '_default': [u'OmegaWiki'], + '_default': [u'Meta'], } self.namespaces[5] = { - '_default': [u'OmegaWiki talk'], + '_default': [u'Meta talk'], } self.namespaces[6] = { '_default': [u'File'], @@ -31,18 +31,6 @@ self.namespaces[17] = { '_default': [u'Expression talk'], } - self.namespaces[18] = { - '_default': [u'Collection'], - } - self.namespaces[19] = { - '_default': [u'Collection talk'], - } - self.namespaces[20] = { - '_default': [u'Relation'], - } - self.namespaces[21] = { - '_default': [u'Relation talk'], - } self.namespaces[22] = { '_default': [u'Portal'], } @@ -67,6 +55,12 @@ self.namespaces[29] = { '_default': [u'NeedsTranslationTo talk'], } + self.namespaces[30] = { + '_default': [u'Partner'], + } + self.namespaces[31] = { + '_default': [u'Partner talk'], + } # On most Wikipedias page names must start with a capital letter, but some # languages don't use this. @@ -77,7 +71,7 @@ return 'www.omegawiki.org' def version(self, code): - return "1.10alpha" + return "1.16alpha" def scriptpath(self, code): return ''

1 0

SVN: [7165] trunk/pywikipedia/commonsdelinker/image_replacer.py
by siebrand＠svn.wikimedia.org 20 Aug '09

20 Aug '09

Revision: 7165 Author: siebrand Date: 2009-08-20 20:53:10 +0000 (Thu, 20 Aug 2009) Log Message: ----------- Revert r7131 in image_replacer.py. Caused: [2009-08-20 20:46:48] Warning! Unable to read replacement log. [2009-08-20 20:46:48] UnboundLocalError: local variable 'response' referenced before assignment Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/image_replacer.py Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-20 20:44:49 UTC (rev 7164) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-20 20:53:10 UTC (rev 7165) @@ -10,7 +10,7 @@ # Distributed under the terms of the MIT license. # __version__ = '$Id$' -import config, wikipedia, query +import config, wikipedia, simplejson import re, time import threadpool import sys, os, signal, traceback @@ -149,19 +149,21 @@ """ Fetch the last 50 revisions using the API """ address = self.site.api_address() - predata = { - 'action': 'query', - 'prop': 'revisions', - 'titles': title.encode('utf-8'), - 'rvprop': 'timestamp|user|comment|content', - 'rvlimit': '50', - } + predata = [ + ('action', 'query'), + ('prop', 'revisions'), + ('titles', title.encode('utf-8')), + ('rvprop', 'timestamp|user|comment|content'), + ('rvlimit', '50'), + ('format', 'json'), + ] if username: - predata['rvexcludeuser'] = username.encode('utf-8') + predata.append(('rvexcludeuser', username.encode('utf-8'))) if since: - predata['rvend'] = since - response, data = query.GetData(predata, self.site, back_response = True) - if data.has_key('error'): + predata.append(('rvend', since)) + response, data = self.site.postForm(address, predata) + data = simplejson.loads(data) + if 'error' in data: raise RuntimeError(data['error']) page = data['query']['pages'].values()[0]

1 0

SVN: [7164] trunk/pywikipedia/commonsdelinker/image_replacer.py
by siebrand＠svn.wikimedia.org 20 Aug '09

20 Aug '09

Revision: 7164 Author: siebrand Date: 2009-08-20 20:44:49 +0000 (Thu, 20 Aug 2009) Log Message: ----------- Return instead of go on an crash. Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/image_replacer.py Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-18 11:39:18 UTC (rev 7163) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-20 20:44:49 UTC (rev 7164) @@ -285,7 +285,7 @@ return except wikipedia.IsRedirectPage: output(u'Warning! %s is a redirect; not reporting replacement!' % old_image) - + return try: page.put(u'%s\n%s' % (template, text), comment = u'This image has been replaced by ' + new_image)

1 0

SVN: [7163] trunk/pywikipedia/family.py
by purodha＠svn.wikimedia.org 18 Aug '09

18 Aug '09

Revision: 7163 Author: purodha Date: 2009-08-18 11:39:18 +0000 (Tue, 18 Aug 2009) Log Message: ----------- Add ksh linktrail and add last 5 new languages to their respective script groups. Modified Paths: -------------- trunk/pywikipedia/family.py Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-17 18:02:44 UTC (rev 7162) +++ trunk/pywikipedia/family.py 2009-08-18 11:39:18 UTC (rev 7163) @@ -2614,6 +2614,7 @@ 'he': u'[a-zא-ת]*', 'it': u'[a-zàèéìòù]*', 'kk': u'[a-zäçéğıïñöşüýа-яёәғіқңөұүһʺʹ]*', + 'ksh': u'[äöüėëĳßəğåůæœça-z]*', 'nl': u'[a-zäöüïëéèéàç]*', 'pt': u'[a-záâàãéêíóôõúüç]*', 'ru': u'[a-zа-я]*', @@ -3093,9 +3094,9 @@ self.language_groups = { # languages using the arabic script (incomplete) 'arab' : [ - 'ar', 'arz', 'ps', 'sd', 'ur', + 'ar', 'arz', 'ps', 'sd', 'ur', 'ckb', # languages using multiple scripts, including arabic - 'kk', 'ku', 'tt', 'ug' + 'kk', 'ku', 'tt', 'ug', 'pnb' ], # languages that use chinese symbols 'chinese': [ @@ -3111,9 +3112,14 @@ # languages using multiple scripts, including cyrillic 'ha', 'kk', 'sh', 'sr', 'tt' ], + # languages that use a greek script + 'grec': [ + 'el', 'grc', 'pnt' + # languages using multiple scripts, including greek + ], # languages that use the latin alphabet 'latin': [ - 'aa', 'af', 'ak', 'als', 'an', 'ang', 'ast', 'ay', 'bar', 'bat-smg', + 'aa', 'ace', 'af', 'ak', 'als', 'an', 'ang', 'ast', 'ay', 'bar', 'bat-smg', 'bcl', 'bi', 'bm', 'br', 'bs', 'ca', 'cbk-zam', 'cdo', 'ceb', 'ch', 'cho', 'chy', 'co', 'crh', 'cs', 'csb', 'cy', 'da', 'de', 'diq', 'dsb', 'ee', 'eml', 'en', 'eo', 'es', 'et', 'eu', 'ext', 'ff', 'fi', 'fiu-vro', @@ -3122,7 +3128,7 @@ 'ie', 'ig', 'ik', 'ilo', 'io', 'is', 'it', 'jbo', 'jv', 'kaa', 'kab', 'kg', 'ki', 'kj', 'kl', 'kr', 'ksh', 'kw', 'la', 'lad', 'lb', 'lg', 'li', 'lij', 'lmo', 'ln', 'lt', 'lv', 'map-bms', 'mg', 'mh', 'mi', 'ms', - 'mt', 'mus', 'na', 'nah', 'nap', 'nds', 'nds-nl', 'ng', 'nl', 'nn', + 'mt', 'mus', 'mwl', 'na', 'nah', 'nap', 'nds', 'nds-nl', 'ng', 'nl', 'nn', 'no', 'nov', 'nrm', 'nv', 'ny', 'oc', 'om', 'pag', 'pam', 'pap', 'pdc', 'pih', 'pl', 'pms', 'pt', 'qu', 'rm', 'rn', 'ro', 'roa-rup', 'roa-tara', 'rw', 'sc', 'scn', 'sco', 'se', 'sg', 'simple', 'sk', 'sl', 'sm', 'sn', @@ -3131,11 +3137,12 @@ 'vec', 'vi', 'vls', 'vo', 'wa', 'war', 'wo', 'xh', 'yo', 'zea', 'zh-min-nan', 'zu', # languages using multiple scripts, including latin - 'az', 'chr', 'ha', 'iu', 'kk', 'ku', 'rmy', 'sh', 'sr', 'tt', 'ug', 'za' + 'az', 'chr', 'ckb', 'ha', 'iu', 'kk', 'ku', 'rmy', 'sh', 'sr', 'tt', + 'ug', 'za' ], # Scandinavian languages 'scand': [ - 'da', 'fo', 'is', 'no', 'sv' + 'da', 'fo', 'is', 'nb', 'nn', 'no', 'sv' ], }

1 0

SVN: [7162] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 17 Aug '09

17 Aug '09

Revision: 7162 Author: alexsh Date: 2009-08-17 18:02:44 +0000 (Mon, 17 Aug 2009) Log Message: ----------- Page()._putPage(): fix sysop cookies in getData Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-17 17:57:39 UTC (rev 7161) +++ trunk/pywikipedia/wikipedia.py 2009-08-17 18:02:44 UTC (rev 7162) @@ -1531,7 +1531,7 @@ params['nocreate'] = 1 # Submit the prepared information try: - response, data = query.GetData(params, self.site(), back_response = True) + response, data = query.GetData(params, self.site(), sysop=sysop, back_response = True) if query.IsString(data): raise KeyError except httplib.BadStatusLine, line:

1 0

SVN: [7161] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 17 Aug '09

17 Aug '09

Revision: 7161 Author: alexsh Date: 2009-08-17 17:57:39 +0000 (Mon, 17 Aug 2009) Log Message: ----------- Page()._putPage(): fix pagedelete and newpages parameters Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-16 18:02:05 UTC (rev 7160) +++ trunk/pywikipedia/wikipedia.py 2009-08-17 17:57:39 UTC (rev 7161) @@ -1525,8 +1525,10 @@ # Which web-site host are we submitting to? if newPage: output(u'Creating page %s' % self.aslink()) + params['createonly'] = 1 else: output(u'Updating page %s' % self.aslink()) + params['nocreate'] = 1 # Submit the prepared information try: response, data = query.GetData(params, self.site(), back_response = True) @@ -1612,6 +1614,7 @@ # without any reason! # raise EditConflict(u'Someone deleted the page.') # No raise, simply define these variables and retry: + params['recreate'] = 1 if self._editTime: params['basetimestamp'] = self._editTime else:

1 0

SVN: [7160] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7160 Author: alexsh Date: 2009-08-16 18:02:05 +0000 (Sun, 16 Aug 2009) Log Message: ----------- typo config.authenticate predata Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-16 18:01:30 UTC (rev 7159) +++ trunk/pywikipedia/query.py 2009-08-16 18:02:05 UTC (rev 7160) @@ -87,10 +87,9 @@ try: jsontext = "Nothing received" if site.hostname() in wikipedia.config.authenticate.keys(): - predata["Content-type"] = "application/x-www-form-urlencoded" - predata["User-agent"] = useragent - data = site.urlEncode(params) - res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, data)) + params["Content-type"] = "application/x-www-form-urlencoded" + params["User-agent"] = useragent + res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, site.urlEncode(params))) jsontext = res.read() elif params['action'] in postAC: res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(sysop=sysop), sysop=sysop)

1 0

SVN: [7159] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7159 Author: alexsh Date: 2009-08-16 18:01:30 +0000 (Sun, 16 Aug 2009) Log Message: ----------- Handle API result ValueError Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-16 16:06:54 UTC (rev 7158) +++ trunk/pywikipedia/wikipedia.py 2009-08-16 18:01:30 UTC (rev 7159) @@ -1545,6 +1545,14 @@ if retry_delay > 30: retry_delay = 30 continue + except ValueError: # API result cannot decode + output(u"Server error encountered; will retry in %i minute%s." + % (retry_delay, retry_delay != 1 and "s" or "")) + time.sleep(60 * retry_delay) + retry_delay *= 2 + if retry_delay > 30: + retry_delay = 30 + continue # If it has gotten this far then we should reset dblagged dblagged = False # Check blocks

1 0

SVN: [7158] trunk/pywikipedia/wikipedia.py
by a_engels＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7158 Author: a_engels Date: 2009-08-16 16:06:54 +0000 (Sun, 16 Aug 2009) Log Message: ----------- list of 'alternative languages' renewed Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-16 14:16:51 UTC (rev 7157) +++ trunk/pywikipedia/wikipedia.py 2009-08-16 16:06:54 UTC (rev 7158) @@ -7041,104 +7041,139 @@ # This code is used by translate() below. def altlang(code): - if code=='aa': + #Amharic + if code in ['aa', 'om']: return ['am'] - if code in ['fa','so']: + #Arab + if code in ['arc', 'arz']: return ['ar'] - if code=='ku': - return ['ar','tr'] - if code=='sk': - return ['cs'] - if code in ['bar','ksh','stq']: + if code == 'kab': + return ['ar', 'fr'] + #Bulgarian + if code in ['cu', 'mk']: + return ['bg', 'sr', 'sh'] + #Czech + if code in ['cs', 'sk']: + return ['cs', 'sk'] + #German + if code in ['als', 'bar', 'ksh']: return ['de'] - if code in ['als','lb']: - return ['de','fr'] - if code=='dsb': - return ['hsb','de'] - if code=='hsb': - return ['dsb','de'] - if code=='io': + if code in ['als', 'lb']: + return ['de', 'fr'] + if code == 'nds': + return ['nds-nl', 'de'] + if code in ['dsb', 'hsb']: + return ['hsb', 'dsb', 'de'] + if code == 'rm': + return ['de', 'it'] + if code == 'stq': + return ['fy', 'de'] + #Greek + if code == 'pnt': + return ['el'] + #Esperanto + if code in ['io', 'nov']: return ['eo'] - if code in ['an','ast','ay','ca','gn','nah','qu']: + #Spanish + if code in ['an', 'ast', 'ay', 'ca', 'ext', 'lad', 'nah', 'nv', 'qu']: return ['es'] - if code == ['cbk-zam']: - return ['es','tl'] - if code=='eu': - return ['es','fr'] - if code in ['glk','mzn']: - return ['fa','ar'] - if code=='gl': - return ['es','pt'] - if code=='lad': - return ['es','he'] - if code in ['br','ht','kab','ln','lo','nrm','wa']: + if code in ['gl', 'gn']: + return ['es', 'pt'] + if code == ['eu']: + return ['es', 'fr'] + if code in ['bcl', 'cbk-zam', 'ceb', 'ilo', 'pag', 'pam', 'tl', 'war']: + return ['es', 'tl'] + #Estonian + if code == 'fiu-vro': + return ['et'] + #Persian (Farsi) + if code in ['glk', 'mzn']: + return ['ar'] + #French + if code in ['bm', 'br', 'ht', 'kab', 'kg', 'ln', 'mg', 'nrm', 'oc', + 'rw', 'sg', 'ty', 'wa']: return ['fr'] - if code in ['ie','oc']: - return ['ie','oc','fr'] - if code in ['co','frp']: - return ['fr','it'] - if code=='yi': - return ['he','de'] - if code=='sa': + if code == 'co': + return ['fr', 'it'] + #Hindi + if code in ['bh', 'pi', 'sa']: return ['hi'] - if code in ['eml','lij','lmo','nap','pms','roa-tara','sc','scn','vec']: + if code in ['ne', 'new']: + return ['ne', 'new', 'hi'] + #Indonesian and Malay + if code in ['ace', 'bug', 'id', 'jv', 'ms', 'su']: + return ['id', 'ms', 'jv'] + if code == 'map-bms': + return ['jv', 'id', 'ms'] + #Inuit languages + if code in ['ik', 'iu']: + return ['iu', 'kl'] + if code == 'kl': + return ['iu', 'da', 'no'] + #Italian + if code in ['eml', 'fur', 'lij', 'lmo', 'nap', 'pms', 'roa-tara', 'sc', + 'scn', 'vec']: return ['it'] - if code=='rm': - return ['it','de','fr'] - if code in ['bat-smg','ltg']: + if code == 'frp': + return ['it', 'fr'] + #Lithuanian + if code in ['bat-smg', 'ltg']: return ['lt'] - if code=='ia': - return ['la','es','fr','it'] - if code=='nds': - return ['nds-nl','de'] - if code=='nds-nl': - return ['nds','nl'] - if code in ['fy','pap','vls','zea']: + #Dutch + if code in ['fy', 'li', 'pap', 'srn', 'vls', 'zea']: return ['nl'] - if code=='li': - return ['nl','de'] - if code=='csb': + if code == ['nds-nl']: + return ['nds', 'nl'] + #Polish + if code in ['csb', 'szl']: return ['pl'] - if code in ['fab','tet']: + #Portuguese + if code in ['fab', 'mwl', 'tet']: return ['pt'] - if code in ['mo','roa-rup']: + #Romanian + if code in ['mo', 'roa-rup']: return ['ro'] - if code in ['av','bxr','cv','hy','lbe','tg','udm','uk','xal']: + #Russian and Belarusian + if code in ['ab', 'av', 'ba', 'bxr', 'ce', 'cv', 'kk', 'ky', 'lbe', 'mdf', + 'mhr', 'myv', 'os', 'sah', 'tg', 'tt', 'udm', 'uk', 'xal']: return ['ru'] - if code in ['be','be-x-old']: - return ['be','be-x-old','ru'] - if code in ['ky','tt','uz']: - return ['kk','tr','ru'] - if code in ['az','diq','tk','ug']: - return ['tr'] - if code in ['ja','minnan','zh','zh-cn']: - return ['zh','zh-tw','zh-classical','zh-cn'] - if code in ['bo','cdo','hak','wuu','za','zh-cdo','zh-classical','zh-tw','zh-yue']: - return ['zh','zh-cn','zh-classical','zh-tw'] - if code=='da': - return ['nb','no'] - if code in ['is','no','nb','nn']: - return ['no','nb','nn','da','sv'] - if code=='sv': - return ['da','no','nb'] - if code=='se': - return ['no','nb','sv','nn','fi','da'] - if code in ['bug','id','jv','map-bms','ms','su']: - return ['id','ms','jv'] - if code in ['bs','hr','sh']: - return ['sh','hr','bs','sr'] - if code in ['mk','sr']: - return ['sh','sr','hr','bs'] - if code in ['ceb','pag','tl','war']: - return ['tl','es'] - if code=='bi': - return ['tpi'] - if code=='tpi': - return ['bi'] - if code == 'new': - return ['ne'] - if code == 'nov': - return ['io','eo'] + if code in ['be', 'be-x-old']: + return ['be', 'be-x-old', 'ru'] + if code == 'kaa': + return ['uz', 'ru'] + #Serbocroatian + if code in ['bs', 'hr', 'sh', 'sr']: + return ['sh', 'hr', 'bs', 'sr'] + #Turkish and Kurdish + if code in ['diq', 'ku']: + return ['ku', 'tr'] + if code == 'ckb': + return ['ku', 'ar'] + #Chinese + if code in ['minnan', 'zh', 'zh-classical', 'zh-min-nan', 'zh-tw']: + return ['zh', 'zh-tw', 'zh-cn', 'zh-classical'] + if code in ['cdo', 'gan', 'hak', 'ii', 'wuu', 'za', 'zh-cdo', 'zh-classical', + 'zh-cn', 'zh-yue']: + return ['zh', 'zh-cn', 'zh-tw', 'zh-classical'] + #Scandinavian languages + if code in ['da', 'sv']: + return ['da', 'no', 'nb', 'sv', 'nn'] + if code in ['fo', 'is']: + return ['da', 'no', 'nb', 'nn', 'sv'] + if code == 'nn': + return ['no', 'nb', 'sv', 'da'] + if code in ['nb', 'no']: + return ['no', 'nb', 'da', 'nn', 'sv'] + if code == 'se': + return ['sv', 'no', 'nb', 'nn', 'fi'] + #Other languages + if code in ['bi', 'tpi']: + return ['bi', 'tpi'] + if code == 'yi': + return ['he', 'de'] + if code in ['ia', 'ie']: + return ['ia', 'la', 'it', 'fr', 'es'] + #Default value return [] def translate(code, xdict):

1 0

SVN: [7157] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7157 Author: alexsh Date: 2009-08-16 14:16:51 +0000 (Sun, 16 Aug 2009) Log Message: ----------- Fix:Page._putPage() getagain in badtoken Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-16 14:14:15 UTC (rev 7156) +++ trunk/pywikipedia/wikipedia.py 2009-08-16 14:16:51 UTC (rev 7157) @@ -1653,7 +1653,7 @@ else: # We might have been using an outdated token output(u"Edit token has failed. Retrying.") - return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop), newToken = True, sysop = sysop) + return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop, getagain = True), newToken = True, sysop = sysop) # I think the error message title was changed from "Wikimedia Error" # to "Wikipedia has a problem", but I'm not sure. Maybe we could # just check for HTTP Status 500 (Internal Server Error)?

1 0

SVN: [7156] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7156 Author: alexsh Date: 2009-08-16 14:14:15 +0000 (Sun, 16 Aug 2009) Log Message: ----------- add config.authenticate into getData Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-16 14:07:10 UTC (rev 7155) +++ trunk/pywikipedia/query.py 2009-08-16 14:14:15 UTC (rev 7156) @@ -86,7 +86,13 @@ while retryCount >= 0: try: jsontext = "Nothing received" - if params['action'] in postAC: + if site.hostname() in wikipedia.config.authenticate.keys(): + predata["Content-type"] = "application/x-www-form-urlencoded" + predata["User-agent"] = useragent + data = site.urlEncode(params) + res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, data)) + jsontext = res.read() + elif params['action'] in postAC: res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(sysop=sysop), sysop=sysop) else: if back_response:

1 0

SVN: [7155] trunk/pywikipedia/families/i18n_family.py
by siebrand＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7155 Author: siebrand Date: 2009-08-16 14:07:10 +0000 (Sun, 16 Aug 2009) Log Message: ----------- Update translatewiki.net namespaces Modified Paths: -------------- trunk/pywikipedia/families/i18n_family.py Modified: trunk/pywikipedia/families/i18n_family.py =================================================================== --- trunk/pywikipedia/families/i18n_family.py 2009-08-16 13:09:09 UTC (rev 7154) +++ trunk/pywikipedia/families/i18n_family.py 2009-08-16 14:07:10 UTC (rev 7155) @@ -93,6 +93,12 @@ self.namespaces[1217] = { '_default': [u'OpenLayers talk'], } + self.namespaces[1218] = { + '_default': [u'FUDforum'], + } + self.namespaces[1219] = { + '_default': [u'FUDforum talk'], + } def version(self, code): return "1.16alpha"

1 0

SVN: [7154] trunk/pywikipedia/families/wikipedia_family.py
by a_engels＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7154 Author: a_engels Date: 2009-08-16 13:09:09 +0000 (Sun, 16 Aug 2009) Log Message: ----------- disambiguation templates in a number of (smaller) languages Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-16 12:32:15 UTC (rev 7153) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-16 13:09:09 UTC (rev 7154) @@ -654,12 +654,15 @@ 'af': [u'Dubbelsinnig', u'Disambig'], 'als': [u'Begriffsklärung', u'Disambig'], 'an': [u'Desambig', u'Disambig'], + 'ang': [u'Disambig', u'Geodis'], 'ar': [u'Disambig', u'توضيح'], 'arc': [u'ܕ'], 'arz': [u'توضيح'], 'ast': [u'Dixebra'], 'av': [u'Неоднозначность'], + 'ay': [u'Desambiguación'], 'az': [u'Dəqiqləşdirmə'], + 'ba': [u'Күп мәғәнәлелек'], 'bar': [u'Begriffsklärung'], 'be': [u'Неадназначнасць', u'Disambig'], 'be-x-old': [u'Неадназначнасць', u'Неадназначнасьць', u'Disambig'], @@ -671,6 +674,7 @@ u'acrònim', u'onomàstica'], 'ceb': [u'Giklaro'], 'cdo': [u'Gì-ngiê'], + 'crh': [u'Çoq manalı', u'Disambig'], 'cs': [u'Rozcestník', u'Rozcestník - 2 znaky', u'Rozcestník - Příjmení', u'Rozcestník - místopisné jméno', u'Disambig', u'Rozcestník - příjmení', u'Rozcestník - sakrální stavba', u'Rozcestník - kostel', @@ -679,6 +683,7 @@ 'cy': [u'Anamrwysedd', u'Disambig', u'Gwahaniaethu'], 'da': [u'Flertydig'], 'de': [u'Begriffsklärung', u'BKS', u'Disambig'], + 'dsb': [u'Wěcejwóznamowosć'], 'el': [u'Disambig', u'Αποσαφ', u'Αποσαφήνιση'], 'en': None, 'eo': [u'Apartigilo', u'Disambig'], @@ -688,11 +693,14 @@ 'ext': [u'Desambiguáncia'], 'fa': [u'ابهام‌زدایی',u'ابهام زدایی'], 'fi': [u'Täsmennyssivu', u'Disambig'], + 'fiu-vro': [u'Täpsüstüslehekülg'], 'fo': [u'Fleiri týdningar'], 'fr': None, 'frp': [u'Homonimos'], + 'fur': [u'Disambiguazion', u'Disambig'], 'fy': [u'Tfs', u'Neibetsjuttings'], 'ga': [u'Idirdhealú', u'Disambig'], + 'gan': [u'扤清楚', u'Disambig'], 'gl': [u'Homónimos', u'Disambig'], 'gv': [u'Reddaghey'], 'haw': [u'Huaʻōlelo puana like'], @@ -724,6 +732,7 @@ 'ln': [u'Bokokani'], 'lt': None, 'mk': [u'Појаснување', u'Disambig', u'Geodis'], + 'mn': [u'Салаа утгатай', u'Салаа', u'Disambig'], 'mo': [u'Дезамбигуйзаре', u'Disambig'], 'ms': [u'Nyahkekaburan', u'Disambig'], 'mt': [u'Diżambigwazzjoni'], @@ -767,6 +776,7 @@ 'tg': [u'Ибҳомзудоӣ', u'Disambig', u'Рафъи ибҳом', u'Disambiguation'], 'th': [u'แก้กำกวม', u'Disambig', u'คำกำกวม'], 'tl': [u'Paglilinaw', u'Disambig'], + 'to': [u'Fakaʻuhingakehe'], 'tr': [u'Anlam ayrım', u'Disambig', u'Anlam ayrımı', u'Kişi adları (anlam ayrımı)', u'Yerleşim yerleri (anlam ayrımı)', u'kısaltmalar (anlam ayrımı)'], @@ -777,6 +787,7 @@ 'vls': [u'Db', u'Dp', u'Dpintro'], 'vo': [u'Telplänov'], 'wa': [u'Omonimeye', u'Disambig'], + 'wo': [u'Bokktekki'], 'yi': [u'באדייטען'], 'zea': [u'dp', u'Deurverwiespagina'], 'zh': [u'Disambig', u'消歧义', u'消歧义页', u'消歧義'],

1 0

SVN: [7153] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 16 Aug '09

16 Aug '09

Revision: 7153 Author: alexsh Date: 2009-08-16 12:32:15 +0000 (Sun, 16 Aug 2009) Log Message: ----------- Update namespaces by update_namespaces.py Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/families/wikisource_family.py trunk/pywikipedia/families/wiktionary_family.py trunk/pywikipedia/family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-15 10:24:24 UTC (rev 7152) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-16 12:32:15 UTC (rev 7153) @@ -59,6 +59,7 @@ self.namespaces[4] = { '_default': [u'Wikipedia', self.namespaces[4]['_default']], 'ar': u'ويكيبيديا', + 'arc': u'ܘܝܩܝܦܕܝܐ', 'arz': u'ويكيبيديا', 'ast': u'Uiquipedia', 'az': u'Vikipediya', @@ -153,6 +154,7 @@ 'am': u'Wikipedia ውይይት', 'an': u'Descusión Wikipedia', 'ar': u'نقاش ويكيبيديا', + 'arc': u'ܘܝܩܝܦܕܝܐ talk', 'arz': u'نقاش ويكيبيديا', 'as': u'Wikipedia বার্তা', 'ast': u'Uiquipedia alderique', @@ -860,7 +862,7 @@ # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementa… self.cross_allowed = [ - 'ab', 'ace', 'af', 'ak', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', + 'ab', 'ace', 'af', 'ak', 'als', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', 'ba', 'bar', 'bcl', 'be-x-old', 'be', 'bg', 'bh', 'bi', 'bm', 'bo', 'bpy', 'bug', 'bxr', 'cbk-zam', 'cdo', 'ce', 'ch', 'chr', 'chy', 'ckb', 'co', 'crh', 'cr', 'csb', 'cu', 'cv', 'cy', 'diq', 'dsb', 'dz', 'ee', 'el', 'eml', 'eu', 'ext', 'fa', 'ff', 'fj', 'fo', 'frp', 'fur', @@ -874,7 +876,7 @@ 'sah', 'sa', 'sc', 'sco', 'sd', 'se', 'sg', 'sh', 'simple', 'si', 'sk', 'sm', 'sn', 'so', 'srn', 'stq', 'st', 'su', 'sw', 'szl', 'ta', 'te', 'tet', 'tg', 'ti', 'tk', 'tn', 'to', 'tpi', 'ts', 'tt', 'tum', 'tw', 'ty', 'udm', 'ug', 'uz', 've', 'vls', 'wa', 'war', 'wo', 'wuu', - 'xal', 'xh', 'yi', 'yo', 'za', 'zea', 'zh', 'zh-classic', 'zh-min-nan', 'zu', + 'xal', 'xh', 'yi', 'yo', 'za', 'zea', 'zh', 'zh-classical', 'zh-min-nan', 'zu', ] # On most Wikipedias page names must start with a capital letter, but some # languages don't use this. @@ -982,7 +984,7 @@ '_default': { 'pt': [102], 'als': [104], - 'ar': [104], + 'ar': [104], 'en': [12], 'es': [104], 'fi': [4], Modified: trunk/pywikipedia/families/wikisource_family.py =================================================================== --- trunk/pywikipedia/families/wikisource_family.py 2009-08-15 10:24:24 UTC (rev 7152) +++ trunk/pywikipedia/families/wikisource_family.py 2009-08-16 12:32:15 UTC (rev 7153) @@ -174,6 +174,7 @@ 'en': u'Author', 'es': u'Página', 'fa': [u'مؤلف', u'Author'], + 'fr': u'Auteur', 'hr': u'Stranica', 'hy': u'Պորտալ', 'it': u'Autore', @@ -193,6 +194,7 @@ 'en': u'Author talk', 'es': u'Página Discusión', 'fa': [u'بحث مؤلف', u'Author talk'], + 'fr': u'Discussion Auteur', 'hr': u'Razgovor o stranici', 'hy': u'Պորտալի քննարկում', 'it': u'Discussioni autore', Modified: trunk/pywikipedia/families/wiktionary_family.py =================================================================== --- trunk/pywikipedia/families/wiktionary_family.py 2009-08-15 10:24:24 UTC (rev 7152) +++ trunk/pywikipedia/families/wiktionary_family.py 2009-08-16 12:32:15 UTC (rev 7153) @@ -16,7 +16,7 @@ 'nl', 'ku', 'ja', 'es', 'id', 'te', 'ro', 'et', 'gl', 'bg', 'ca', 'ar', 'vo', 'uk', 'fa', 'is', 'sr', 'af', 'sw', 'scn', 'fy', 'th', 'br', 'oc', 'simple', 'li', 'cs', 'he', 'sl', 'hy', - 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb' + 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb', 'kk', 'ml', 'ky', 'hr', 'wo', 'kn', 'ang', 'eo', 'hi', 'gn', 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', 'sd', 'ug', 'ti', 'tl', 'mk', 'ka', 'an', 'my', 'gu', 'km', @@ -89,6 +89,7 @@ 'sk': u'Wikislovník', 'sl': u'Wikislovar', 'sr': u'Викиречник', + 'tk': u'Wikisözlük', 'tr': u'Vikisözlük', 'tt': u'Wiktionary', 'uk': u'Вікісловник', @@ -203,6 +204,7 @@ 'te': u'Wiktionary చర్చ', 'tg': u'Баҳси Wiktionary', 'th': u'คุยเรื่องWiktionary', + 'tk': u'Wikisözlük talk', 'tl': u'Usapang Wiktionary', 'tr': u'Vikisözlük tartışma', 'tt': u'Wiktionary bäxäse', Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-15 10:24:24 UTC (rev 7152) +++ trunk/pywikipedia/family.py 2009-08-16 12:32:15 UTC (rev 7153) @@ -894,12 +894,14 @@ }, 4: { '_default': u'Project', + 'arc': u'ܘܝܩܝܦܕܝܐ', 'cs': u'Wikizdroje', 'ko': u'위키문헌', 'uk': u'Вікіцитати', }, 5: { '_default': u'Project talk', + 'arc': u'ܘܝܩܝܦܕܝܐ talk', 'cs': u'Wikizdroje diskuse', 'gan': u'Wikipedia talk', 'ko': u'위키문헌토론',

1 0

SVN: [7152] trunk/pywikipedia/families/wikipedia_family.py
by siebrand＠svn.wikimedia.org 15 Aug '09

15 Aug '09

Revision: 7152 Author: siebrand Date: 2009-08-15 10:24:24 +0000 (Sat, 15 Aug 2009) Log Message: ----------- #2825805: Update interwiki sort order for Urdu further. Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 16:11:38 UTC (rev 7151) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-15 10:24:24 UTC (rev 7152) @@ -944,7 +944,7 @@ 'pl': self.alphabetic, 'simple': self.alphabetic, 'te': ['en', 'hi', 'kn', 'ta', 'ml'], - 'ur': ['ar', 'fa', 'en'], + 'ur': ['ar', 'fa', 'en'] + self.alphabetic, 'vi': self.alphabetic_revised, 'yi': ['en', 'he', 'de'] }

1 0

SVN: [7151] trunk/pywikipedia/families/wiktionary_family.py
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7151 Author: siebrand Date: 2009-08-14 16:11:38 +0000 (Fri, 14 Aug 2009) Log Message: ----------- Fix indentation error Modified Paths: -------------- trunk/pywikipedia/families/wiktionary_family.py Modified: trunk/pywikipedia/families/wiktionary_family.py =================================================================== --- trunk/pywikipedia/families/wiktionary_family.py 2009-08-14 15:24:44 UTC (rev 7150) +++ trunk/pywikipedia/families/wiktionary_family.py 2009-08-14 16:11:38 UTC (rev 7151) @@ -10,23 +10,23 @@ family.Family.__init__(self) self.name = 'wiktionary' - self.languages_by_size = [ - 'fr', 'en', 'tr', 'lt', 'vi', 'ru', 'io', 'zh', 'el', 'pl', - 'fi', 'no', 'hu', 'it', 'ta', 'sv', 'de', 'ko', 'lo', 'pt', - 'nl', 'ku', 'ja', 'es', 'id', 'te', 'ro', 'et', 'gl', 'bg', - 'ca', 'ar', 'vo', 'uk', 'fa', 'is', 'sr', 'af', 'sw', 'scn', - 'fy', 'th', 'br', 'oc', 'simple', 'li', 'cs', 'he', 'sl', 'hy', - 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb' - 'kk', 'ml', 'ky', 'hr', 'wo', 'kn', 'ang', 'eo', 'hi', 'gn', - 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', - 'sd', 'ug', 'ti', 'tl', 'mk', 'ka', 'an', 'my', 'gu', 'km', - 'lv', 'cy', 'ts', 'qu', 'eu', 'fo', 'bs', 'am', 'rw', 'mr', - 'su', 'chr', 'mn', 'nah', 'om', 'ie', 'yi', 'be', 'iu', 'mg', - 'sh', 'gd', 'nn', 'bn', 'zu', 'si', 'pa', 'mt', 'dv', 'tpi', - 'mi', 'roa-rup', 'jv', 'tg', 'ps', 'ik', 'so', 'uz', 'ha', 'gv', - 'ss', 'kw', 'sa', 'ay', 'na', 'jbo', 'ne', 'tn', 'sm', 'sg', - 'lb', 'ks', 'fj', 'ln', 'za', 'dz', 'als', - ] + self.languages_by_size = [ + 'fr', 'en', 'tr', 'lt', 'vi', 'ru', 'io', 'zh', 'el', 'pl', + 'fi', 'no', 'hu', 'it', 'ta', 'sv', 'de', 'ko', 'lo', 'pt', + 'nl', 'ku', 'ja', 'es', 'id', 'te', 'ro', 'et', 'gl', 'bg', + 'ca', 'ar', 'vo', 'uk', 'fa', 'is', 'sr', 'af', 'sw', 'scn', + 'fy', 'th', 'br', 'oc', 'simple', 'li', 'cs', 'he', 'sl', 'hy', + 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb' + 'kk', 'ml', 'ky', 'hr', 'wo', 'kn', 'ang', 'eo', 'hi', 'gn', + 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', + 'sd', 'ug', 'ti', 'tl', 'mk', 'ka', 'an', 'my', 'gu', 'km', + 'lv', 'cy', 'ts', 'qu', 'eu', 'fo', 'bs', 'am', 'rw', 'mr', + 'su', 'chr', 'mn', 'nah', 'om', 'ie', 'yi', 'be', 'iu', 'mg', + 'sh', 'gd', 'nn', 'bn', 'zu', 'si', 'pa', 'mt', 'dv', 'tpi', + 'mi', 'roa-rup', 'jv', 'tg', 'ps', 'ik', 'so', 'uz', 'ha', 'gv', + 'ss', 'kw', 'sa', 'ay', 'na', 'jbo', 'ne', 'tn', 'sm', 'sg', + 'lb', 'ks', 'fj', 'ln', 'za', 'dz', 'als', + ] if config.SSL_connection: for lang in self.languages_by_size:

1 0

SVN: [7150] trunk/pywikipedia
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7150 Author: siebrand Date: 2009-08-14 15:24:44 +0000 (Fri, 14 Aug 2009) Log Message: ----------- Update family.py and wikipedia_family.py for ace and mwl Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 15:07:59 UTC (rev 7149) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 15:24:44 UTC (rev 7150) @@ -147,6 +147,7 @@ self.namespaces[5] = { '_default': [u'Wikipedia talk', self.namespaces[5]['_default']], 'ab': u'Обсуждение Wikipedia', + 'ace': u'Pembicaraan Wikipedia', 'af': u'Wikipediabespreking', 'als': u'Wikipedia Diskussion', 'am': u'Wikipedia ውይይት', @@ -257,6 +258,7 @@ 'mr': u'विकिपीडिया चर्चा', 'ms': u'Perbincangan Wikipedia', 'mt': [u'Wikipedija diskussjoni', u'Wikipedija diskuti'], + 'mwl': u'Wikipedia cumbersa', 'myv': u'Википедиясь кортамось', 'mzn': u'بحث Wikipedia', 'nah': [u'Wikipedia tēixnāmiquiliztli', u'Wikipedia Discusión'], Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-14 15:07:59 UTC (rev 7149) +++ trunk/pywikipedia/family.py 2009-08-14 15:24:44 UTC (rev 7150) @@ -168,6 +168,7 @@ -1: { '_default': u'Special', 'ab': u'Служебная', + 'ace': u'Istimewa', 'af': u'Spesiaal', 'als': u'Spezial', 'am': u'ልዩ', @@ -346,6 +347,7 @@ 1: { '_default': u'Talk', 'ab': u'Обсуждение', + 'ace': u'Pembicaraan', 'af': u'Bespreking', 'als': u'Diskussion', 'am': u'ውይይት', @@ -455,6 +457,7 @@ 'mr': u'चर्चा', 'ms': u'Perbincangan', 'mt': u'Diskussjoni', + 'mwl': u'Cumbersa', 'myv': u'Кортамо', 'mzn': u'بحث', 'nah': [u'Tēixnāmiquiliztli', u'Discusión'], @@ -527,6 +530,7 @@ 2: { '_default': u'User', 'ab': u'Участник', + 'ace': u'Pengguna', 'af': u'Gebruiker', 'als': u'Benutzer', 'am': u'አባል', @@ -635,6 +639,7 @@ 'mr': u'सदस्य', 'ms': u'Pengguna', 'mt': u'Utent', + 'mwl': u'Outelizador', 'myv': u'Теиця', 'mzn': u'کاربر', 'nah': [u'Tlatequitiltilīlli', u'Usuario'], @@ -708,6 +713,7 @@ 3: { '_default': u'User talk', 'ab': u'Обсуждение участника', + 'ace': u'Pembicaraan Pengguna', 'af': u'Gebruikerbespreking', 'als': u'Benutzer Diskussion', 'am': u'አባል ውይይት', @@ -815,6 +821,7 @@ 'mr': u'सदस्य चर्चा', 'ms': u'Perbincangan Pengguna', 'mt': u'Diskussjoni utent', + 'mwl': u'Cumbersa outelizador', 'myv': u'Теицянь кортамось', 'mzn': u'بحث کاربر', 'nah': [u'Tlatequitiltilīlli tēixnāmiquiliztli', u'Usuario Discusión'], @@ -904,6 +911,7 @@ 6: { '_default': [u'File', u'Image'], 'ab': [u'Файл', u'Изображение'], + 'ace': u'Berkas', 'af': [u'Lêer', u'Beeld'], 'als': [u'Datei', u'Bild'], 'am': u'ስዕል', @@ -1014,6 +1022,7 @@ 'mr': u'चित्र', 'ms': [u'Fail', u'Imej'], 'mt': u'Stampa', + 'mwl': u'Fexeiro', 'myv': u'Артовкс', 'mzn': [u'پرونده', u'تصویر'], 'nah': [u'Īxiptli', u'Imagen'], @@ -1086,6 +1095,7 @@ 7: { '_default': [u'File talk', u'Image talk'], 'ab': [u'Обсуждение файла', u'Обсуждение изображения'], + 'ace': u'Pembicaraan Berkas', 'af': [u'Lêerbespreking', u'Beeldbespreking'], 'als': [u'Datei Diskussion', u'Bild Diskussion'], 'am': u'ስዕል ውይይት', @@ -1195,6 +1205,7 @@ 'mr': u'चित्र चर्चा', 'ms': [u'Perbincangan Fail', u'Perbincangan Imej'], 'mt': u'Diskussjoni stampa', + 'mwl': u'Cumbersa fexeiro', 'myv': u'Артовксто кортамось', 'mzn': [u'بحث پرونده', u'بحث تصویر'], 'nah': [u'Īxiptli tēixnāmiquiliztli', u'Imagen Discusión'], @@ -1301,6 +1312,7 @@ 'ml': u'മീഡിയവിക്കി', 'mn': u'МедиаВики', 'mr': u'मिडियाविकी', + 'mwl': u'Biqui', 'mzn': u'مدیاویکی', 'nah': u'Huiquimedia', 'new': u'मिडियाविकि', @@ -1324,6 +1336,7 @@ 9: { '_default': u'MediaWiki talk', 'ab': u'Обсуждение MediaWiki', + 'ace': u'Pembicaraan MediaWiki', 'af': u'MediaWikibespreking', 'als': u'MediaWiki Diskussion', 'am': u'መልዕክት ውይይት', @@ -1431,6 +1444,7 @@ 'mr': u'मिडियाविकी चर्चा', 'ms': u'Perbincangan MediaWiki', 'mt': u'Diskussjoni MediaWiki', + 'mwl': u'Cumbersa Biqui', 'myv': u'MediaWiki-нь кортамось', 'mzn': u'بحث مدیاویکی', 'nah': [u'Huiquimedia tēixnāmiquiliztli', u'MediaWiki Discusión'], @@ -1502,6 +1516,7 @@ 10: { '_default': u'Template', 'ab': u'Шаблон', + 'ace': u'Templat', 'af': u'Sjabloon', 'als': u'Vorlage', 'am': u'መልጠፊያ', @@ -1606,6 +1621,7 @@ 'mr': u'साचा', 'ms': u'Templat', 'mt': u'Mudell', + 'mwl': u'Modelo', 'myv': u'ЛопаПарцун', 'mzn': u'الگو', 'nah': [u'Nemachiyōtīlli', u'Plantilla'], @@ -1673,6 +1689,7 @@ 11: { '_default': u'Template talk', 'ab': u'Обсуждение шаблона', + 'ace': u'Pembicaraan Templat', 'af': u'Sjabloonbespreking', 'als': u'Vorlage Diskussion', 'am': u'መልጠፊያ ውይይት', @@ -1781,6 +1798,7 @@ 'mr': u'साचा चर्चा', 'ms': u'Perbincangan Templat', 'mt': u'Diskussjoni mudell', + 'mwl': u'Cumbersa Modelo', 'myv': u'ЛопаПарцундо кортамось', 'mzn': u'بحث الگو', 'nah': [u'Nemachiyōtīlli tēixnāmiquiliztli', u'Plantilla Discusión'], @@ -1852,6 +1870,7 @@ 12: { '_default': u'Help', 'ab': u'Справка', + 'ace': u'Bantuan', 'af': u'Hulp', 'als': u'Hilfe', 'am': u'እርዳታ', @@ -1959,6 +1978,7 @@ 'mr': [u'सहाय्य', u'साहाय्य'], 'ms': u'Bantuan', 'mt': [u'Għajnuna', u'Ghajnuna'], + 'mwl': u'Ajuda', 'myv': u'Лезкс', 'mzn': u'راهنما', 'nah': [u'Tēpalēhuiliztli', u'Ayuda'], @@ -2030,6 +2050,7 @@ 13: { '_default': u'Help talk', 'ab': u'Обсуждение справки', + 'ace': u'Pembicaraan Bantuan', 'af': u'Hulpbespreking', 'als': u'Hilfe Diskussion', 'am': u'እርዳታ ውይይት', @@ -2139,6 +2160,7 @@ 'mr': [u'सहाय्य चर्चा', u'साहाय्य चर्चा'], 'ms': u'Perbincangan Bantuan', 'mt': [u'Diskussjoni għajnuna', u'Diskussjoni ghajnuna'], + 'mwl': u'Cumbersa ajuda', 'myv': u'Лезкстэ кортамось', 'mzn': u'بحث راهنما', 'nah': [u'Tēpalēhuiliztli tēixnāmiquiliztli', u'Ayuda Discusión'], @@ -2211,6 +2233,7 @@ 14: { '_default': u'Category', 'ab': u'Категория', + 'ace': u'Kategori', 'af': u'Kategorie', 'als': u'Kategorie', 'am': u'መደብ', @@ -2319,6 +2342,7 @@ 'mr': u'वर्ग', 'ms': u'Kategori', 'mt': u'Kategorija', + 'mwl': u'Catadorie', 'myv': u'Категория', 'mzn': u'رده', 'nah': [u'Neneuhcāyōtl', u'Categoría'], @@ -2392,6 +2416,7 @@ 15: { '_default': u'Category talk', 'ab': u'Обсуждение категории', + 'ace': u'Pembicaraan Kategori', 'af': u'Kategoriebespreking', 'als': u'Kategorie Diskussion', 'am': u'መደብ ውይይት', @@ -2501,6 +2526,7 @@ 'mr': u'वर्ग चर्चा', 'ms': u'Perbincangan Kategori', 'mt': u'Diskussjoni kategorija', + 'mwl': u'Cumbersa catadorie', 'myv': u'Категориядо кортамось', 'mzn': u'بحث رده', 'nah': [u'Neneuhcāyōtl tēixnāmiquiliztli', u'Categoría Discusión'],

1 0

SVN: [7149] trunk/pywikipedia/families
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7149 Author: siebrand Date: 2009-08-14 15:07:59 +0000 (Fri, 14 Aug 2009) Log Message: ----------- Update self.languages_by_size for Wikimedia projects. Used wikimedia_sites.py as source Modified Paths: -------------- trunk/pywikipedia/families/wikibooks_family.py trunk/pywikipedia/families/wikinews_family.py trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/families/wikiquote_family.py trunk/pywikipedia/families/wikisource_family.py trunk/pywikipedia/families/wiktionary_family.py Modified: trunk/pywikipedia/families/wikibooks_family.py =================================================================== --- trunk/pywikipedia/families/wikibooks_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wikibooks_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -11,16 +11,16 @@ self.name = 'wikibooks' self.languages_by_size = [ - 'en', 'de', 'fr', 'hu', 'pt', 'ja', 'es', 'nl', 'it', 'pl', - 'he', 'sq', 'fi', 'vi', 'ru', 'cs', 'hr', 'th', 'zh', 'da', - 'sv', 'mk', 'tr', 'sr', 'fa', 'tl', 'id', 'is', 'ca', 'ta', - 'ar', 'no', 'ko', 'eo', 'ka', 'simple', 'bg', 'gl', 'lt', 'ro', + 'en', 'de', 'fr', 'zh', 'no', 'hu', 'pt', 'ja', 'es', 'it', + 'nl', 'pl', 'sq', 'he', 'fi', 'vi', 'ru', 'cs', 'hr', 'da', + 'sv', 'mk', 'th', 'tr', 'sr', 'fa', 'tl', 'ca', 'id', 'is', + 'ta', 'ar', 'ko', 'eo', 'ka', 'simple', 'bg', 'gl', 'lt', 'ro', 'sk', 'la', 'el', 'ia', 'ang', 'sl', 'et', 'mr', 'cv', 'ur', - 'oc', 'ml', 'uk', 'ms', 'eu', 'fy', 'lv', 'hi', 'ie', 'tg', - 'bn', 'hy', 'af', 'az', 'te', 'pa', 'bs', 'ky', 'be', 'sa', - 'zh-min-nan', 'ast', 'cy', 'tt', 'mg', 'km', 'si', 'ku', 'co', 'sw', - 'tk', 'ne', 'als', 'uz', 'vo', 'su', 'na', 'mn', 'lb', 'kn', - 'ps', 'xh', 'kk', 'za', 'nds', 'wa', 'zu', 'my', + 'oc', 'ml', 'ms', 'uk', 'eu', 'fy', 'ie', 'hi', 'tg', 'bn', + 'hy', 'af', 'az', 'te', 'pa', 'bs', 'ky', 'be', 'sa', 'zh-min-nan', + 'ast', 'cy', 'tt', 'mg', 'si', 'ku', 'co', 'tk', 'sw', 'ne', + 'als', 'su', 'uz', 'vo', 'na', 'kn', 'mn', 'kk', 'za', 'nds', + 'zu', 'my', ] if config.SSL_connection: Modified: trunk/pywikipedia/families/wikinews_family.py =================================================================== --- trunk/pywikipedia/families/wikinews_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wikinews_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -11,9 +11,9 @@ self.name = 'wikinews' self.languages_by_size = [ - 'en', 'pl', 'de', 'sr', 'it', 'fr', 'pt', 'es', 'zh', 'sv', - 'ja', 'ru', 'nl', 'he', 'fi', 'sd', 'ar', 'cs', 'no', 'uk', - 'ca', 'hu', 'ro', 'th', 'bs', 'bg', 'ta', 'tr' + 'en', 'sr', 'pl', 'it', 'de', 'fr', 'pt', 'es', 'zh', 'sv', + 'ja', 'ru', 'nl', 'he', 'fi', 'ar', 'sd', 'cs', 'no', 'uk', + 'hu', 'ca', 'ro', 'th', 'bs', 'bg', 'ta', 'tr' ] if config.SSL_connection: Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -11,32 +11,32 @@ self.name = 'wikipedia' self.languages_by_size = [ - 'en', 'de', 'fr', 'pl', 'ja', 'it', 'nl', 'pt', 'es', 'ru', - 'sv', 'zh', 'no', 'fi', 'ca', 'uk', 'tr', 'cs', 'hu', 'ro', - 'vo', 'eo', 'da', 'sk', 'id', 'ko', 'ar', 'he', 'vi', 'lt', - 'sl', 'sr', 'bg', 'et', 'fa', 'hr', 'new', 'simple', 'ht', 'nn', - 'gl', 'th', 'te', 'el', 'ms', 'eu', 'ceb', 'hi', 'mk', 'ka', - 'la', 'lb', 'bs', 'br', 'is', 'az', 'sq', 'cy', 'mr', 'bpy', - 'sh', 'tl', 'lv', 'pms', 'be-x-old', 'bn', 'jv', 'oc', 'ta', 'io', - 'be', 'an', 'nds', 'su', 'scn', 'nap', 'ku', 'af', 'fy', 'ast', - 'sw', 'wa', 'zh-yue', 'bat-smg', 'qu', 'ur', 'cv', 'ml', 'ksh', 'war', - 'tg', 'ga', 'vec', 'roa-tara', 'uz', 'gd', 'gu', 'pam', 'kn', 'mi', - 'yi', 'yo', 'lmo', 'glk', 'nah', 'zh-min-nan', 'co', 'hsb', 'roa-rup', 'als', - 'kk', 'ia', 'li', 'hy', 'gan', 'sah', 'tt', 'tk', 'sa', 'wuu', - 'nds-nl', 'fo', 'os', 'vls', 'fiu-vro', 'arz', 'nrm', 'bcl', 'am', 'rm', - 'pag', 'map-bms', 'dv', 'mn', 'gv', 'se', 'diq', 'fur', 'ne', 'sco', - 'bar', 'lij', 'nov', 'bh', 'mt', 'ilo', 'pi', 'mzn', 'zh-classical', 'km', - 'ug', 'csb', 'frp', 'lad', 'si', 'pdc', 'kw', 'ang', 'haw', 'sc', - 'mg', 'to', 'kv', 'szl', 'pa', 'ps', 'ie', 'my', 'gn', 'ln', - 'hif', 'stq', 'wo', 'jbo', 'arc', 'crh', 'tpi', 'ty', 'ext', 'cbk-zam', - 'ky', 'eml', 'zea', 'srn', 'ay', 'myv', 'hak', 'pap', 'ig', 'kg', - 'so', 'or', 'kab', 'lo', 'rmy', 'ba', 'ce', 'sm', 'udm', 'av', - 'ks', 'nv', 'dsb', 'cu', 'got', 'kaa', 'tet', 'sd', 'bo', 'mdf', - 'iu', 'na', 'bm', 'as', 'cdo', 'chr', 'ee', 'om', 'pnt', 'pih', - 'kl', 'zu', 'ab', 'ti', 'ts', 'ss', 'za', 'bi', 'cr', 'dz', - 'xal', 've', 'ch', 'ha', 'xh', 'tn', 'bug', 'st', 'bxr', 'ik', - 'rw', 'chy', 'tw', 'ak', 'fj', 'ny', 'ff', 'sn', 'sg', 'lbe', - 'rn', 'ki', 'lg', 'tum', 'ng', 'mhr', 'ckb', 'pnb', 'mwl', 'ace' + 'en', 'de', 'fr', 'pl', 'ja', 'it', 'nl', 'es', 'pt', 'ru', + 'sv', 'zh', 'no', 'fi', 'ca', 'uk', 'hu', 'tr', 'cs', 'ro', + 'vo', 'eo', 'da', 'sk', 'id', 'ko', 'ar', 'vi', 'he', 'lt', + 'sr', 'sl', 'bg', 'et', 'fa', 'hr', 'simple', 'new', 'ht', 'nn', + 'gl', 'th', 'ms', 'el', 'te', 'eu', 'ceb', 'hi', 'mk', 'ka', + 'la', 'br', 'bs', 'lb', 'is', 'az', 'cy', 'sq', 'mr', 'sh', + 'bpy', 'lv', 'tl', 'pms', 'be-x-old', 'jv', 'bn', 'oc', 'ta', 'io', + 'be', 'an', 'scn', 'nds', 'su', 'nap', 'ku', 'sw', 'af', 'fy', + 'ast', 'zh-yue', 'bat-smg', 'wa', 'war', 'ur', 'qu', 'ml', 'ksh', 'cv', + 'tg', 'ga', 'vec', 'roa-tara', 'gu', 'gd', 'uz', 'pam', 'kn', 'yi', + 'mi', 'lmo', 'yo', 'zh-min-nan', 'nah', 'glk', 'co', 'hsb', 'roa-rup', 'kk', + 'als', 'ia', 'li', 'hy', 'sah', 'bcl', 'gan', 'tt', 'tk', 'sa', + 'nds-nl', 'wuu', 'os', 'fo', 'vls', 'arz', 'fiu-vro', 'nrm', 'am', 'pag', + 'rm', 'map-bms', 'dv', 'mn', 'gv', 'se', 'diq', 'fur', 'ne', 'sco', + 'bar', 'lij', 'nov', 'bh', 'mt', 'ilo', 'pi', 'mzn', 'zh-classical', 'csb', + 'ug', 'km', 'frp', 'lad', 'si', 'kw', 'ang', 'sc', 'pdc', 'haw', + 'mg', 'to', 'kv', 'my', 'szl', 'pa', 'ps', 'myv', 'ie', 'gn', + 'hif', 'ln', 'stq', 'jbo', 'wo', 'arc', 'crh', 'tpi', 'ty', 'ext', + 'cbk-zam', 'ky', 'hak', 'eml', 'zea', 'srn', 'ay', 'pap', 'so', 'kg', + 'ig', 'or', 'kab', 'lo', 'udm', 'rmy', 'ba', 'dsb', 'ce', 'sm', + 'av', 'nv', 'cu', 'ks', 'tet', 'got', 'kaa', 'bo', 'sd', 'mdf', + 'iu', 'na', 'bm', 'as', 'cdo', 'kl', 'pnt', 'chr', 'ee', 'om', + 'pih', 'ab', 'zu', 'ti', 'ts', 'ss', 'za', 'bi', 'ha', 'cr', + 'dz', 'xal', 've', 'ch', 'xh', 'tn', 'bug', 'ik', 'bxr', 'rw', + 'st', 'tw', 'chy', 'ak', 'fj', 'ny', 'ff', 'sn', 'sg', 'lbe', + 'lg', 'rn', 'ki', 'tum', 'ng', 'mhr', 'ckb', 'pnb', 'mwl', 'ace' ] if config.SSL_connection: Modified: trunk/pywikipedia/families/wikiquote_family.py =================================================================== --- trunk/pywikipedia/families/wikiquote_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wikiquote_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -11,13 +11,13 @@ self.name = 'wikiquote' self.languages_by_size = [ - 'en', 'it', 'de', 'pl', 'pt', 'sk', 'ru', 'bg', 'es', 'bs', - 'sl', 'tr', 'he', 'lt', 'fr', 'cs', 'zh', 'hu', 'uk', 'id', - 'fa', 'sv', 'el', 'no', 'nl', 'ja', 'fi', 'eo', 'hy', 'et', - 'nn', 'ca', 'simple', 'ka', 'ar', 'ku', 'hr', 'ko', 'ro', 'gl', + 'en', 'it', 'de', 'pl', 'pt', 'sk', 'ru', 'es', 'bg', 'bs', + 'sl', 'tr', 'he', 'lt', 'fr', 'cs', 'zh', 'fa', 'hu', 'uk', + 'id', 'sv', 'el', 'no', 'nl', 'ja', 'fi', 'eo', 'hy', 'et', + 'nn', 'ca', 'simple', 'ka', 'ar', 'ku', 'ko', 'hr', 'ro', 'gl', 'sr', 'ml', 'li', 'is', 'th', 'af', 'te', 'da', 'sq', 'vi', 'eu', 'az', 'la', 'br', 'hi', 'be', 'ast', 'uz', 'ang', 'zh-min-nan', - 'lb', 'mr', 'su', 'gu', 'ur', 'ta', 'wo', 'kn', 'ky', 'cy', + 'lb', 'mr', 'gu', 'su', 'ur', 'ta', 'wo', 'kn', 'ky', 'cy', 'am', 'co', 'kk', ] Modified: trunk/pywikipedia/families/wikisource_family.py =================================================================== --- trunk/pywikipedia/families/wikisource_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wikisource_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -13,9 +13,9 @@ self.languages_by_size = [ 'en', 'pt', 'fr', 'zh', 'es', 'de', 'ru', 'he', 'it', 'ar', 'fa', 'hu', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'tr', - 'nl', 'sv', 'ko', 'sl', 'sr', 'la', 'ja', 'uk', 'li', 'el', - 'ml', 'is', 'bs', 'id', 'ca', 'bn', 'yi', 'az', 'hy', 'no', - 'mk', 'da', 'vi', 'ta', 'et', 'bg', 'lt', 'kn', 'gl', 'cy', + 'nl', 'sv', 'sl', 'ko', 'sr', 'ja', 'la', 'uk', 'el', 'li', + 'ml', 'is', 'bs', 'id', 'ca', 'bn', 'az', 'yi', 'hy', 'mk', + 'no', 'da', 'vi', 'et', 'ta', 'bg', 'lt', 'kn', 'gl', 'cy', 'sk', 'zh-min-nan', 'fo', ] Modified: trunk/pywikipedia/families/wiktionary_family.py =================================================================== --- trunk/pywikipedia/families/wiktionary_family.py 2009-08-14 14:59:01 UTC (rev 7148) +++ trunk/pywikipedia/families/wiktionary_family.py 2009-08-14 15:07:59 UTC (rev 7149) @@ -10,25 +10,24 @@ family.Family.__init__(self) self.name = 'wiktionary' - self.languages_by_size = [ - 'fr', 'en', 'tr', 'lt', 'vi', 'ru', 'io', 'zh', 'el', 'pl', - 'fi', 'no', 'hu', 'it', 'ta', 'sv', 'de', 'ko', 'lo', 'pt', - 'nl', 'ku', 'es', 'ja', 'id', 'te', 'et', 'ro', 'gl', 'bg', - 'vo', 'ar', 'uk', 'ca', 'is', 'sr', 'fa', 'af', 'li', 'scn', - 'sw', 'fy', 'th', 'br', 'oc', 'he', 'cs', 'simple', 'sl', 'hy', - 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'ur', 'hsb', 'kk', 'wa', - 'ml', 'ky', 'hr', 'wo', 'tk', 'ang', 'eo', 'kn', 'hi', 'gn', - 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', - 'sd', 'ug', 'ti', 'tl', 'mk', 'an', 'my', 'gu', 'ka', 'km', - 'lv', 'ts', 'cy', 'qu', 'fo', 'bs', 'am', 'rw', 'eu', 'mr', - 'su', 'chr', 'mn', 'nah', 'ie', 'yi', 'om', 'be', 'gd', 'sh', - 'iu', 'mg', 'nn', 'zu', 'bn', 'pa', 'si', 'mt', 'tpi', 'dv', - 'mi', 'jv', 'ps', 'roa-rup', 'tg', 'so', 'ik', 'ha', 'gv', 'uz', - 'ss', 'kw', 'sa', 'ay', 'na', 'ne', 'jbo', 'tn', 'sm', 'sg', - 'lb', 'ks', 'fj', 'ln', 'za', 'dz', 'als', - ] + self.languages_by_size = [ + 'fr', 'en', 'tr', 'lt', 'vi', 'ru', 'io', 'zh', 'el', 'pl', + 'fi', 'no', 'hu', 'it', 'ta', 'sv', 'de', 'ko', 'lo', 'pt', + 'nl', 'ku', 'ja', 'es', 'id', 'te', 'ro', 'et', 'gl', 'bg', + 'ca', 'ar', 'vo', 'uk', 'fa', 'is', 'sr', 'af', 'sw', 'scn', + 'fy', 'th', 'br', 'oc', 'simple', 'li', 'cs', 'he', 'sl', 'hy', + 'sq', 'tt', 'la', 'zh-min-nan', 'da', 'ast', 'tk', 'ur', 'wa', 'hsb' + 'kk', 'ml', 'ky', 'hr', 'wo', 'kn', 'ang', 'eo', 'hi', 'gn', + 'ga', 'ia', 'az', 'co', 'sk', 'csb', 'st', 'ms', 'nds', 'kl', + 'sd', 'ug', 'ti', 'tl', 'mk', 'ka', 'an', 'my', 'gu', 'km', + 'lv', 'cy', 'ts', 'qu', 'eu', 'fo', 'bs', 'am', 'rw', 'mr', + 'su', 'chr', 'mn', 'nah', 'om', 'ie', 'yi', 'be', 'iu', 'mg', + 'sh', 'gd', 'nn', 'bn', 'zu', 'si', 'pa', 'mt', 'dv', 'tpi', + 'mi', 'roa-rup', 'jv', 'tg', 'ps', 'ik', 'so', 'uz', 'ha', 'gv', + 'ss', 'kw', 'sa', 'ay', 'na', 'jbo', 'ne', 'tn', 'sm', 'sg', + 'lb', 'ks', 'fj', 'ln', 'za', 'dz', 'als', + ] - if config.SSL_connection: for lang in self.languages_by_size: self.langs[lang] = None

1 0

SVN: [7148] trunk/pywikipedia
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7148 Author: siebrand Date: 2009-08-14 14:59:01 +0000 (Fri, 14 Aug 2009) Log Message: ----------- Changes for added Wikimedia projects: * Wikipedia: ace, ckb, mwl, pnb * Wikinews: tr Modified Paths: -------------- trunk/pywikipedia/families/wikinews_family.py trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/family.py Modified: trunk/pywikipedia/families/wikinews_family.py =================================================================== --- trunk/pywikipedia/families/wikinews_family.py 2009-08-14 09:10:42 UTC (rev 7147) +++ trunk/pywikipedia/families/wikinews_family.py 2009-08-14 14:59:01 UTC (rev 7148) @@ -13,7 +13,7 @@ self.languages_by_size = [ 'en', 'pl', 'de', 'sr', 'it', 'fr', 'pt', 'es', 'zh', 'sv', 'ja', 'ru', 'nl', 'he', 'fi', 'sd', 'ar', 'cs', 'no', 'uk', - 'ca', 'hu', 'ro', 'th', 'bs', 'bg', 'ta', + 'ca', 'hu', 'ro', 'th', 'bs', 'bg', 'ta', 'tr' ] if config.SSL_connection: @@ -53,6 +53,7 @@ 'ru': u'Викиновости', 'sr': u'Викивести', 'th': u'วิกิข่าว', + 'tr': u'Vikihaber', 'uk': u'ВікіНовини', } self.namespaces[5] = { @@ -81,6 +82,7 @@ 'sv': u'Wikinewsdiskussion', 'ta': u'Wikinews பேச்சு', 'th': u'คุยเรื่องวิกิข่าว', + 'tr': u'Vikihaber tartışma', 'uk': u'Обговорення ВікіНовини', } Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 09:10:42 UTC (rev 7147) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 14:59:01 UTC (rev 7148) @@ -36,7 +36,7 @@ 'kl', 'zu', 'ab', 'ti', 'ts', 'ss', 'za', 'bi', 'cr', 'dz', 'xal', 've', 'ch', 'ha', 'xh', 'tn', 'bug', 'st', 'bxr', 'ik', 'rw', 'chy', 'tw', 'ak', 'fj', 'ny', 'ff', 'sn', 'sg', 'lbe', - 'rn', 'ki', 'lg', 'tum', 'ng', 'mhr', + 'rn', 'ki', 'lg', 'tum', 'ng', 'mhr', 'ckb', 'pnb', 'mwl', 'ace' ] if config.SSL_connection: @@ -858,17 +858,17 @@ # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementa… self.cross_allowed = [ - 'ab', 'af', 'ak', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', + 'ab', 'ace', 'af', 'ak', 'am', 'ang', 'arc', 'arz', 'as', 'ast,', 'av', 'ay', 'az', 'ba', 'bar', 'bcl', 'be-x-old', 'be', 'bg', 'bh', 'bi', 'bm', 'bo', 'bpy', 'bug', 'bxr', - 'cbk-zam', 'cdo', 'ce', 'ch', 'chr', 'chy', 'co', 'crh', 'cr', 'csb', 'cu', 'cv', 'cy', + 'cbk-zam', 'cdo', 'ce', 'ch', 'chr', 'chy', 'ckb', 'co', 'crh', 'cr', 'csb', 'cu', 'cv', 'cy', 'diq', 'dsb', 'dz', 'ee', 'el', 'eml', 'eu', 'ext', 'fa', 'ff', 'fj', 'fo', 'frp', 'fur', 'ga', 'gan', 'glk', 'gn', 'got', 'gu', 'gv', 'ha', 'hak', 'haw', 'hif', 'hi', 'hsb', 'ht', 'hu', 'hy', 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'iow', 'is', 'iu', 'ja', 'jbo', 'jv', 'kaa', 'kab', 'ka', 'kg', 'ki', 'kl', 'km', 'kn', 'ko', 'ks', 'ku', 'kv', 'kw', 'ky', 'lad', 'lb', 'lbe', 'lg', 'li', 'lij', 'lmo', 'ln', 'lo', 'lv', - 'map-bms', 'mdf', 'mg', 'mhr', 'mk', 'mn', 'ms', 'mt', 'myv', 'my', 'mzn', + 'map-bms', 'mdf', 'mg', 'mhr', 'mk', 'mn', 'ms', 'mt', 'mwl', 'myv', 'my', 'mzn', 'nah', 'na', 'nap', 'nds-nl', 'ne', 'new', 'ng', 'nl', 'nov', 'nrm', 'nv', 'ny', 'om', 'or', 'os', - 'pam', 'pap', 'pa', 'pdc', 'pnt', 'ps', 'qu', 'rm', 'rmy', 'rn', 'roa-rup', 'roa-tara', 'rw', + 'pam', 'pap', 'pa', 'pdc', 'pnb', 'pnt', 'ps', 'qu', 'rm', 'rmy', 'rn', 'roa-rup', 'roa-tara', 'rw', 'sah', 'sa', 'sc', 'sco', 'sd', 'se', 'sg', 'sh', 'simple', 'si', 'sk', 'sm', 'sn', 'so', 'srn', 'stq', 'st', 'su', 'sw', 'szl', 'ta', 'te', 'tet', 'tg', 'ti', 'tk', 'tn', 'to', 'tpi', 'ts', 'tt', 'tum', 'tw', 'ty', 'udm', 'ug', 'uz', 've', 'vls', 'wa', 'war', 'wo', 'wuu', @@ -879,10 +879,9 @@ self.nocapitalize = ['jbo',] - # A revised sorting order worked out on http://meta.wikimedia.org/wiki/Interwiki_sorting_order self.alphabetic_revised = [ - 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', + 'af', 'ak', 'ace', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', 'frp', 'as', 'ast', 'gn', 'av', 'ay', 'az', 'id', 'ms', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'jv', 'su', 'ba', 'be', 'be-x-old', 'bh', 'bcl', 'bi', 'bar', 'bo', 'bs', 'br', 'bug', 'bg', 'bxr', @@ -896,14 +895,14 @@ 'kn', 'kr', 'pam', 'ka', 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'sw', 'kv', 'kg', 'ht', 'ku', 'kj', 'lad', 'lbe', 'lo', 'la', 'lv', 'to', 'lb', 'lt', 'lij', 'li', 'ln', 'jbo', 'lg', 'lmo', 'hu', - 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'arz', 'mzn', 'cdo', 'mdf', + 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'arz', 'mzn', 'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'fj', 'nl', 'nds-nl', 'cr', 'ne', 'new', 'ja', 'nap', 'ce', 'pih', 'no', 'nb', 'nn', 'nrm', 'nov', - 'ii', 'oc', 'mhr', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pi', 'pag', 'pap', + 'ii', 'oc', 'mhr', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pnb', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'nds', 'pl', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'ru', 'sah', 'se', 'sa', 'sg', 'sc', 'za', 'sco', 'stq', 'st', 'tn', 'sq', 'scn', 'si', 'simple', - 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so', 'srn', 'sr', 'sh', 'fi', + 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'fi', 'sv', 'tl', 'ta', 'kab', 'roa-tara', 'tt', 'te', 'tet', 'th', 'vi', 'ti', 'tg', 'tpi', 'tokipona', 'tp', 'chr', 'chy', 've', 'tr', 'tk', 'tw', 'udm', 'uk', 'ur', 'ug', 'vec', 'vo', 'fiu-vro', 'wa', Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-14 09:10:42 UTC (rev 7147) +++ trunk/pywikipedia/family.py 2009-08-14 14:59:01 UTC (rev 7148) @@ -11,7 +11,7 @@ self.name = None # Updated from http://meta.wikimedia.org/wiki/Interwiki_sorting_order self.alphabetic = [ - 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', + 'af', 'ak', 'ace', 'als', 'am', 'ang', 'ab', 'ar', 'an', 'arc', 'roa-rup', 'frp', 'as', 'ast', 'gn', 'av', 'ay', 'az', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'ba', 'be', 'be-x-old', 'bh', 'bcl', 'bi', 'bar', 'bo', 'bs', 'br', 'bg', 'bxr', 'ca', 'cv', 'ceb', 'cs', @@ -25,14 +25,14 @@ 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'sw', 'kv', 'kg', 'ht', 'ku', 'kj', 'lad', 'lbe', 'lo', 'la', 'lv', 'lb', 'lt', 'lij', 'li', 'ln', 'jbo', 'lg', 'lmo', 'hu', 'mk', 'mg', 'ml', 'mt', 'mi', 'mr', - 'arz', 'mzn', 'ms', 'cdo', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', + 'arz', 'mzn', 'ms', 'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'na', 'fj', 'nl', 'nds-nl', 'cr', 'ne', 'new', 'ja', 'nap', 'ce', 'pih', 'no', 'nb', 'nn', 'nrm', 'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng', - 'hz', 'uz', 'pa', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'tpi', + 'hz', 'uz', 'pa', 'pnb', 'pi', 'pag', 'pap', 'ps', 'km', 'pms', 'tpi', 'nds', 'pl', 'tokipona', 'tp', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'ru', 'sah', 'se', 'sm', 'sa', 'sg', 'sc', 'za', 'sco', 'stq', 'st', 'tn', 'sq', 'scn', 'si', - 'simple', 'sd', 'ss', 'sk', 'cu', 'sl', 'szl', 'so', 'srn', 'sr', + 'simple', 'sd', 'ss', 'sk', 'cu', 'sl', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'su', 'fi', 'sv', 'tl', 'ta', 'kab', 'roa-tara', 'tt', 'te', 'tet', 'th', 'ti', 'tg', 'to', 'chr', 'chy', 've', 'tr', 'tk', 'tw', 'udm', 'bug', 'uk', 'ur', 'ug', 'vec', 'vi', 'vo', 'fiu-vro',

1 0

SVN: [7147] trunk/pywikipedia/families/wikipedia_family.py
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7147 Author: siebrand Date: 2009-08-14 09:10:42 +0000 (Fri, 14 Aug 2009) Log Message: ----------- #2814700: update 'ro' disambiguation templates Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 09:07:43 UTC (rev 7146) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 09:10:42 UTC (rev 7147) @@ -739,7 +739,7 @@ 'pt': [u'Desambiguação', u'Disambig', u'Desambig'], 'qu': [u"Sut'ichana qillqa", u'Disambig', u'SJM'], 'rmy': [u'Dudalipen'], - 'ro': [u'Dezambiguizare', u'Disambig', u'Hndis', u'Dez'], + 'ro': [u'Dezambiguizare', u'Disambig', u'Hndis', u'Dez', u'Dezamb'], 'ru': [u'Disambig', u'Неоднозначность', u'неоднозначность', u'Многозначность', u'Фамилия'], 'scn': [u'Disambigua', u'Disambig', u'Sigla2', u'Sigla3'],

1 0

SVN: [7146] trunk/pywikipedia/families/wikipedia_family.py
by siebrand＠svn.wikimedia.org 14 Aug '09

14 Aug '09

Revision: 7146 Author: siebrand Date: 2009-08-14 09:07:43 +0000 (Fri, 14 Aug 2009) Log Message: ----------- #2825805: Update interwiki sort order for Urdu Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-13 02:24:51 UTC (rev 7145) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-14 09:07:43 UTC (rev 7146) @@ -943,6 +943,7 @@ 'pl': self.alphabetic, 'simple': self.alphabetic, 'te': ['en', 'hi', 'kn', 'ta', 'ml'], + 'ur': ['ar', 'fa', 'en'], 'vi': self.alphabetic_revised, 'yi': ['en', 'he', 'de'] }

1 0

SVN: [7145] trunk/pywikipedia/login.py
by alexsh＠svn.wikimedia.org 12 Aug '09

12 Aug '09

Revision: 7145 Author: alexsh Date: 2009-08-13 02:24:51 +0000 (Thu, 13 Aug 2009) Log Message: ----------- Add -clean to enable cleanup site session...(tired to handle badtoken always) Modified Paths: -------------- trunk/pywikipedia/login.py Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2009-08-12 21:29:35 UTC (rev 7144) +++ trunk/pywikipedia/login.py 2009-08-13 02:24:51 UTC (rev 7145) @@ -165,6 +165,15 @@ else: if api: response, data = query.GetData(predata, self.site, back_response = True) + if data['login']['result'] != "Success": + faildInfo = data['login']['result'] + #if faildInfo == "NotExists": + # + #elif faildInfo == "WrongPass": + # + #elif faildInfo == "Throttled": + # + return False else: response, data = self.site.postData(address, self.site.urlEncode(predata)) if self.verbose: @@ -290,6 +299,7 @@ os.remove(filename) except: pass + wikipedia.output('%s is logged out.' % self.site) return True return False @@ -303,6 +313,7 @@ logall = False forceLogin = False verbose = False + cleanAll = clean = False for arg in wikipedia.handleArgs(): if arg.startswith("-pass"): @@ -310,6 +321,8 @@ password = wikipedia.input(u'Password for all accounts:', password = True) else: password = arg[6:] + elif arg == "-clean": + clean = True elif arg == "-sysop": sysop = True elif arg == "-all": @@ -333,17 +346,30 @@ for lang in namedict[familyName].iterkeys(): try: site = wikipedia.getSite(lang, familyName) - if not forceLogin and site.loggedInAs(sysop = sysop): - wikipedia.output(u'Already logged in on %s' % site) + loginMan = LoginManager(password, sysop = sysop, site = site, verbose=verbose) + if clean: + if os.path.exists(wikipedia.config.datafilepath('login-data', + '%s-%s-%s-login.data' % (familyName, lang, namedict[familyName][lang]))): + loginMan.logout() else: - loginMan = LoginManager(password, sysop = sysop, site = site, verbose=verbose) - loginMan.login() + if not forceLogin and site.loggedInAs(sysop = sysop): + wikipedia.output(u'Already logged in on %s' % site) + else: + loginMan.login() except wikipedia.NoSuchSite: wikipedia.output(lang+ u'.' + familyName + u' is not a valid site, please remove it from your config') else: - loginMan = LoginManager(password, sysop = sysop, verbose=verbose) - loginMan.login() + if clean: + try: + site = wikipedia.getSite() + lgm = LoginManager(site = site) + lgm.logout() + except wikipedia.NoSuchSite: + pass + else: + loginMan = LoginManager(password, sysop = sysop, verbose=verbose) + loginMan.login() if __name__ == "__main__": try:

1 0

SVN: [7144] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 12 Aug '09

12 Aug '09

Revision: 7144 Author: alexsh Date: 2009-08-12 21:29:35 +0000 (Wed, 12 Aug 2009) Log Message: ----------- change simplejson to internal json for 2.6 (take exception to simplejson) Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-11 16:35:49 UTC (rev 7143) +++ trunk/pywikipedia/query.py 2009-08-12 21:29:35 UTC (rev 7144) @@ -25,7 +25,13 @@ __version__ = '$Id$' # -import wikipedia, simplejson, urllib, time +import wikipedia, urllib, time +try: + #For Python 2.6 newer + import json +except ImportError: + import simplejson as json + def GetData(params, site = None, verbose = False, useAPI = True, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): """Get data from the query api, and convert it into a data object @@ -91,15 +97,15 @@ # This will also work, but all unicode strings will need to be converted from \u notation # decodedObj = eval( jsontext ) if back_response: - return res, simplejson.loads( jsontext ) + return res, json.loads( jsontext ) else: - return simplejson.loads( jsontext ) + return json.loads( jsontext ) except ValueError, error: retryCount -= 1 wikipedia.output(u"Error downloading data: %s" % error) wikipedia.output(u"Request %s:%s" % (site.lang, path)) - wikipedia.debugDump('ApiGetDataParse', site, str(error) + '\n%s' % path, jsontext) + wikipedia.debugDump('ApiGetDataParse', site, str(error) + '\n%s\n%s' % (site.hostname(), path), jsontext) lastError = error if retryCount >= 0: wikipedia.output(u"Retrying in %i seconds..." % retry_idle_time)

1 0

SVN: [7143] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 11 Aug '09

11 Aug '09

Revision: 7143 Author: alexsh Date: 2009-08-11 16:35:49 +0000 (Tue, 11 Aug 2009) Log Message: ----------- typo Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-11 10:59:58 UTC (rev 7142) +++ trunk/pywikipedia/wikipedia.py 2009-08-11 16:35:49 UTC (rev 7143) @@ -1625,7 +1625,7 @@ continue elif errorCode == 'contenttoobig': # 'contenttoobig':"The content you supplied exceeds the article size limit of \$1 kilobytes", - raise LongPageError(len(params['text']), int(data['error']['indo'][59:-10])) + raise LongPageError(len(params['text']), int(data['error']['info'][59:-10])) elif errorCode in ['protectedpage', 'customcssjsprotected', 'cascadeprotected', 'protectednamespace', 'protectednamespace-interface']: # 'protectedpage':"The ``\$1'' right is required to edit this page" # 'cascadeprotected':"The page you're trying to edit is protected because it's included in a cascade-protected page"

1 0

SVN: [7142] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 11 Aug '09

11 Aug '09

Revision: 7142 Author: alexsh Date: 2009-08-11 10:59:58 +0000 (Tue, 11 Aug 2009) Log Message: ----------- ImagePage().fileUrl(): typy Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-11 10:53:06 UTC (rev 7141) +++ trunk/pywikipedia/wikipedia.py 2009-08-11 10:59:58 UTC (rev 7142) @@ -3081,7 +3081,8 @@ } imagedata = query.GetData(params, self.site(), encodeTitle = False) try: - return imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] + url = imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] + return url # urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) # m = urlR.search(self.getImagePageHtml())

1 0

SVN: [7141] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 11 Aug '09

11 Aug '09

Revision: 7141 Author: alexsh Date: 2009-08-11 10:53:06 +0000 (Tue, 11 Aug 2009) Log Message: ----------- *Page()._putPage(): change to use API, old move to _putPageOld() --All possible error in API mode should be handled --need to set config.use_api to enable this, otherwise it will use _putPageOld() Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-10 19:21:56 UTC (rev 7140) +++ trunk/pywikipedia/wikipedia.py 2009-08-11 10:53:06 UTC (rev 7141) @@ -1453,6 +1453,233 @@ def _putPage(self, text, comment=None, watchArticle=False, minorEdit=True, newPage=False, token=None, newToken=False, sysop=False, captcha=None, botflag=True, maxTries=-1): + """Upload 'text' as new content of Page by API + + Don't use this directly, use put() instead. + + """ + try: + if config.use_api and self.site().versionnumber() >= 13: + apitest = self.site().api_address() + del apitest + else: + raise NotImplementedError #No enable api or version not support + except NotImplementedError: + return self._putPageOld(text, comment, watchArticle, minorEdit, + newPage, token, newToken, sysop, captcha, botflag, maxTries) + + retry_attempt = 1 + retry_delay = 1 + dblagged = False + params = { + 'action': 'edit', + 'title': self.title(), + 'text': self._encodeArg(text, 'text'), + 'summary': self._encodeArg(comment, 'summary'), + } + + if token: + params['token'] = token + else: + params['token'] = self.site().getToken(sysop = sysop) + + # Add server lag parameter (see config.py for details) + if config.maxlag: + params['maxlag'] = str(config.maxlag) + + if self._editTime: + params['basetimestamp'] = self._editTime + else: + params['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) + + if self._startTime: + params['starttimestamp'] = self._startTime + else: + params['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) + + if botflag: + params['bot'] = 1 + + if minorEdit: + params['minor'] = 1 + else: + params['notminor'] = 1 + + if watchArticle: + params['watch'] = 1 + #else: + # params['unwatch'] = 1 + + if captcha: + params['captchaid'] = captcha['id'] + params['captchaword'] = captcha['answer'] + + while True: + if (maxTries == 0): + raise MaxTriesExceededError() + maxTries -= 1 + # Check whether we are not too quickly after the previous + # putPage, and wait a bit until the interval is acceptable + if not dblagged: + put_throttle() + # Which web-site host are we submitting to? + if newPage: + output(u'Creating page %s' % self.aslink()) + else: + output(u'Updating page %s' % self.aslink()) + # Submit the prepared information + try: + response, data = query.GetData(params, self.site(), back_response = True) + if query.IsString(data): + raise KeyError + except httplib.BadStatusLine, line: + raise PageNotSaved('Bad status line: %s' % line.line) + except ServerError: + output(u''.join(traceback.format_exception(*sys.exc_info()))) + retry_attempt += 1 + if retry_attempt > config.maxretries: + raise + output(u'Got a server error when putting %s; will retry in %i minute%s.' % (self.aslink(), retry_delay, retry_delay != 1 and "s" or "")) + time.sleep(60 * retry_delay) + retry_delay *= 2 + if retry_delay > 30: + retry_delay = 30 + continue + # If it has gotten this far then we should reset dblagged + dblagged = False + # Check blocks + self.site().checkBlocks(sysop = sysop) + # A second text area means that an edit conflict has occured. + if response.status == 500: + output(u"Server error encountered; will retry in %i minute%s." + % (retry_delay, retry_delay != 1 and "s" or "")) + time.sleep(60 * retry_delay) + retry_delay *= 2 + if retry_delay > 30: + retry_delay = 30 + continue + if data.has_key('error'): + #All available error key in edit mode: (from ApiBase.php) + # 'noimageredirect-anon':"Anonymous users can't create image redirects", + # 'noimageredirect':"You don't have permission to create image redirects", + # 'filtered':"The filter callback function refused your edit", + # 'noedit-anon':"Anonymous users can't edit pages", + # 'noedit':"You don't have permission to edit pages", + # 'emptypage':"Creating new, empty pages is not allowed", + # 'badmd5':"The supplied MD5 hash was incorrect", + # 'notext':"One of the text, appendtext, prependtext and undo parameters must be set", + # 'emptynewsection':'Creating empty new sections is not possible.', + # 'revwrongpage':"r\$1 is not a revision of ``\$2''", + # 'undofailure':'Undo failed due to conflicting intermediate edits', + + #for debug only + #------------------------ + if verbose: + output("error occured, result:%s\nstatus:%s\nresponse:%s" % (data, response.status, response.reason)) + faked = params + del faked['text'], faked['format'] + output("OriginalData:%s" % faked) + del faked + #------------------------ + errorCode = data['error']['code'] + #cannot handle longpageerror and PageNoSave yet + if errorCode == 'maxlag' or response.status == 503: + # server lag; Mediawiki recommends waiting 5 seconds + # and retrying + if verbose: + output(data, newline=False) + output(u"Pausing 5 seconds due to database server lag.") + dblagged = True + time.sleep(5) + continue + elif errorCode == 'editconflict': + # 'editconflict':"Edit conflict detected", + raise EditConflict(u'An edit conflict has occured.') + elif errorCode == 'spamdetected': + # 'spamdetected':"Your edit was refused because it contained a spam fragment: ``\$1''", + raise SpamfilterError(data['error']['info'][62:-2]) + elif errorCode == 'pagedeleted': + # 'pagedeleted':"The page has been deleted since you fetched its timestamp", + # Make sure your system clock is correct if this error occurs + # without any reason! + # raise EditConflict(u'Someone deleted the page.') + # No raise, simply define these variables and retry: + if self._editTime: + params['basetimestamp'] = self._editTime + else: + params['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) + + if self._startTime: + params['starttimestamp'] = self._startTime + else: + params['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) + continue + elif errorCode == 'readonly': + # 'readonly':"The wiki is currently in read-only mode" + output(u"The database is currently locked for write access; will retry in %i minute%s." + % (retry_delay, retry_delay != 1 and "s" or "")) + time.sleep(60 * retry_delay) + retry_delay *= 2 + if retry_delay > 30: + retry_delay = 30 + continue + elif errorCode == 'contenttoobig': + # 'contenttoobig':"The content you supplied exceeds the article size limit of \$1 kilobytes", + raise LongPageError(len(params['text']), int(data['error']['indo'][59:-10])) + elif errorCode in ['protectedpage', 'customcssjsprotected', 'cascadeprotected', 'protectednamespace', 'protectednamespace-interface']: + # 'protectedpage':"The ``\$1'' right is required to edit this page" + # 'cascadeprotected':"The page you're trying to edit is protected because it's included in a cascade-protected page" + # 'customcssjsprotected': "You're not allowed to edit custom CSS and JavaScript pages" + # 'protectednamespace': "You're not allowed to edit pages in the ``\$1'' namespace" + # 'protectednamespace-interface':"You're not allowed to edit interface messages" + # + # The page is locked. This should have already been + # detected when getting the page, but there are some + # reasons why this didn't work, e.g. the page might be + # locked via a cascade lock. + try: + # Page is locked - try using the sysop account, unless we're using one already + if sysop:# Unknown permissions error + raise LockedPage() + else: + self.site().forceLogin(sysop = True) + output(u'Page is locked, retrying using sysop account.') + return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = True), sysop = True) + except NoUsername: + raise LockedPage() + elif errorCode == 'badtoken': + if newToken: + output(u"Edit token has failed. Giving up.") + else: + # We might have been using an outdated token + output(u"Edit token has failed. Retrying.") + return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop), newToken = True, sysop = sysop) + # I think the error message title was changed from "Wikimedia Error" + # to "Wikipedia has a problem", but I'm not sure. Maybe we could + # just check for HTTP Status 500 (Internal Server Error)? + else: + output("Unknown Error. API Error code:%s" % data['error']['code'] ) + output("Information:%s" %data['error']['info']) + else: + if data['edit']['result'] == u"Success": + # + # The status code for update page completed in ordinary mode is 302 - Found + # But API is always 200 - OK because it only send "success" back in string. + # if the page update is successed, we need to return code 302 for cheat script who + # using status code + # + return 302, response.reason, data + + solve = self.site().solveCaptcha(data) + if solve: + return self._putPage(text, comment, watchArticle, minorEdit, newPage, token, newToken, sysop, captcha=solve) + + return response.status, response.reason, data + + + def _putPageOld(self, text, comment=None, watchArticle=False, minorEdit=True, + newPage=False, token=None, newToken=False, sysop=False, + captcha=None, botflag=True, maxTries=-1): """Upload 'text' as new content of Page by filling out the edit form. Don't use this directly, use put() instead. @@ -1625,14 +1852,14 @@ else: self.site().forceLogin(sysop = True) output(u'Page is locked, retrying using sysop account.') - return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = True), sysop = True) + return self._putPageOld(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = True), sysop = True) except NoUsername: raise LockedPage() if not newToken and "<textarea" in data: ##if "<textarea" in data: # for debug use only, if badtoken still happen # We might have been using an outdated token output(u"Changing page has failed. Retrying.") - return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop, getagain = True), newToken = True, sysop = sysop) + return self._putPageOld(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop, getagain = True), newToken = True, sysop = sysop) # I think the error message title was changed from "Wikimedia Error" # to "Wikipedia has a problem", but I'm not sure. Maybe we could # just check for HTTP Status 500 (Internal Server Error)? @@ -1683,7 +1910,7 @@ ## output('%s' % data) # WHY? solve = self.site().solveCaptcha(data) if solve: - return self._putPage(text, comment, watchArticle, minorEdit, newPage, token, newToken, sysop, captcha=solve) + return self._putPageOld(text, comment, watchArticle, minorEdit, newPage, token, newToken, sysop, captcha=solve) # We are expecting a 302 to the action=view page. I'm not sure why this was removed in r5019 if data.strip() != u"": @@ -2795,9 +3022,7 @@ raise NoPage(u'API Error, nothing found in the APIs') # We don't know the page's id, if any other better idea please change it - pageid = data.keys()[0] - nickdata = data[pageid][u'revisions'] - return nickdata + return data[data.keys()[0]][u'revisions'] class ImagePage(Page): """A subclass of Page representing an image descriptor wiki page. @@ -2856,7 +3081,7 @@ } imagedata = query.GetData(params, self.site(), encodeTitle = False) try: - url=imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] + return imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] # urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) # m = urlR.search(self.getImagePageHtml()) @@ -5152,7 +5377,8 @@ self._mediawiki_messages = _dict([(tag['name'].lower(), tag['*']) for tag in datas]) except KeyError: - output('API get messages had some error, retrying by ordinary.') + if verbose: + output('API get messages had some error, retrying by ordinary.') api = False continue except NotImplementedError: @@ -5207,8 +5433,7 @@ key = key.lower() try: - value = self._mediawiki_messages[key] - return value + return self._mediawiki_messages[key] except KeyError: raise KeyError("MediaWiki key '%s' does not exist on %s" % (key, self))

1 0

SVN: [7140] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 10 Aug '09

10 Aug '09

Revision: 7140 Author: alexsh Date: 2009-08-10 19:21:56 +0000 (Mon, 10 Aug 2009) Log Message: ----------- *catlib.py: add config lock. *wiktionary_family.py: update ns by update_namespaces.py Modified Paths: -------------- trunk/pywikipedia/catlib.py trunk/pywikipedia/families/wiktionary_family.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-09 08:40:29 UTC (rev 7139) +++ trunk/pywikipedia/catlib.py 2009-08-10 19:21:56 UTC (rev 7140) @@ -179,7 +179,7 @@ This should not be used outside of this module. """ try: - if self.site().versionnumber() >= 11: + if wikipedia.config.use_api and self.site().versionnumber() >= 11: api_url = self.site().api_address() del api_url else: @@ -206,14 +206,14 @@ if currentPageOffset: params['cmcontinue'] = currentPageOffset - wikipedia.output('Getting [[%s]] list from %s by API...' + wikipedia.output('Getting [[%s]] list from %s...' % (self.title(), currentPageOffset[:-1])) # cmcontinue last key is '|' elif startFrom: params['cmstart'] = startFrom - wikipedia.output('Getting [[%s]] list starting at %s by API...' + wikipedia.output('Getting [[%s]] list starting at %s...' % (self.title(), startFrom)) else: - wikipedia.output('Getting [[%s]] by API...' % self.title()) + wikipedia.output('Getting [[%s]]...' % self.title()) wikipedia.get_throttle() data = query.GetData(params, self.site()) Modified: trunk/pywikipedia/families/wiktionary_family.py =================================================================== --- trunk/pywikipedia/families/wiktionary_family.py 2009-08-09 08:40:29 UTC (rev 7139) +++ trunk/pywikipedia/families/wiktionary_family.py 2009-08-10 19:21:56 UTC (rev 7140) @@ -300,7 +300,7 @@ self.namespaces[104] = { 'bs': u'Dodatak', 'cy': u'WiciSawrws', - 'de': u'WikiSaurus', + 'de': u'Thesaurus', 'en': u'Index', 'fr': u'Portail', 'pl': u'Portal', @@ -312,7 +312,7 @@ self.namespaces[105] = { 'bs': u'Razgovor o Dodatku', 'cy': u'Sgwrs WiciSawrws', - 'de': u'WikiSaurus Diskussion', + 'de': u'Thesaurus Diskussion', 'en': u'Index talk', 'fr': u'Discussion Portail', 'pl': u'Dyskusja portalu',

1 0

SVN: [7139] trunk/pywikipedia/catlib.py
by alexsh＠svn.wikimedia.org 09 Aug '09

09 Aug '09

Revision: 7139 Author: alexsh Date: 2009-08-09 08:40:29 +0000 (Sun, 09 Aug 2009) Log Message: ----------- catlib.py: typo Modified Paths: -------------- trunk/pywikipedia/catlib.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-09 08:36:24 UTC (rev 7138) +++ trunk/pywikipedia/catlib.py 2009-08-09 08:40:29 UTC (rev 7139) @@ -223,7 +223,7 @@ if isCatTitle(memb['title'], self.site()): yield SUBCATEGORY, Category(self.site(), memb['title']) elif memb['ns'] == 6 and self.site().image_namespace() in memb['title']: - yield ARTICLE, wikipedia.ImagePage(self.site(), title) + yield ARTICLE, wikipedia.ImagePage(self.site(), memb['title']) else: yield ARTICLE, wikipedia.Page(self.site(), memb['title']) # try to find a link to the next list page

1 0

SVN: [7138] trunk/pywikipedia/catlib.py
by alexsh＠svn.wikimedia.org 09 Aug '09

09 Aug '09

Revision: 7138 Author: alexsh Date: 2009-08-09 08:36:24 +0000 (Sun, 09 Aug 2009) Log Message: ----------- typo config from wikipedia.py Modified Paths: -------------- trunk/pywikipedia/catlib.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-09 06:00:26 UTC (rev 7137) +++ trunk/pywikipedia/catlib.py 2009-08-09 08:36:24 UTC (rev 7138) @@ -198,10 +198,10 @@ 'cmprop': 'title',#|ids|sortkey|timestamp', #'': '', } - if config.special_page_limit > 500: + if wikipedia.config.special_page_limit > 500: params['cmlimit'] = 500 else: - params['cmlimit'] = config.special_page_limit + params['cmlimit'] = wikipedia.config.special_page_limit if currentPageOffset:

1 0

SVN: [7137] trunk/pywikipedia/catlib.py
by alexsh＠svn.wikimedia.org 09 Aug '09

09 Aug '09

Revision: 7137 Author: alexsh Date: 2009-08-09 06:00:26 +0000 (Sun, 09 Aug 2009) Log Message: ----------- *Category()._parseCategory():typo cmlimit Modified Paths: -------------- trunk/pywikipedia/catlib.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-09 05:18:21 UTC (rev 7136) +++ trunk/pywikipedia/catlib.py 2009-08-09 06:00:26 UTC (rev 7137) @@ -196,9 +196,14 @@ 'list': 'categorymembers', 'cmtitle': self.title(), 'cmprop': 'title',#|ids|sortkey|timestamp', - #'cmlimit': config.special_page_limit, #'': '', } + if config.special_page_limit > 500: + params['cmlimit'] = 500 + else: + params['cmlimit'] = config.special_page_limit + + if currentPageOffset: params['cmcontinue'] = currentPageOffset wikipedia.output('Getting [[%s]] list from %s by API...'

1 0

SVN: [7136] trunk/pywikipedia/catlib.py
by alexsh＠svn.wikimedia.org 09 Aug '09

09 Aug '09

Revision: 7136 Author: alexsh Date: 2009-08-09 05:18:21 +0000 (Sun, 09 Aug 2009) Log Message: ----------- catlib.py *Category()._parseCategory(): Change use API query(list=categorymembers), could except to old way(moved to _oldParseCategory) if version not support or NotImplementedError (tested for 12 hours in [[ja:WP:BOTREQ]]) Modified Paths: -------------- trunk/pywikipedia/catlib.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-08 15:28:21 UTC (rev 7135) +++ trunk/pywikipedia/catlib.py 2009-08-09 05:18:21 UTC (rev 7136) @@ -166,6 +166,69 @@ def _parseCategory(self, purge=False, startFrom=None): """ + Yields all articles and subcategories that are in this category by API. + + Set startFrom to a string which is the title of the page to start from. + + Yielded results are tuples in the form (tag, page) where tag is one + of the constants ARTICLE and SUBCATEGORY, and title is the Page or Category + object. + + Note that results of this method need not be unique. + + This should not be used outside of this module. + """ + try: + if self.site().versionnumber() >= 11: + api_url = self.site().api_address() + del api_url + else: + raise NotImplementedError # version not support + except NotImplementedError: + for tag, page in self._oldParseCategory(purge, startFrom): + yield tag, page + return + + currentPageOffset = None + while True: + params = { + 'action': 'query', + 'list': 'categorymembers', + 'cmtitle': self.title(), + 'cmprop': 'title',#|ids|sortkey|timestamp', + #'cmlimit': config.special_page_limit, + #'': '', + } + if currentPageOffset: + params['cmcontinue'] = currentPageOffset + wikipedia.output('Getting [[%s]] list from %s by API...' + % (self.title(), currentPageOffset[:-1])) # cmcontinue last key is '|' + elif startFrom: + params['cmstart'] = startFrom + wikipedia.output('Getting [[%s]] list starting at %s by API...' + % (self.title(), startFrom)) + else: + wikipedia.output('Getting [[%s]] by API...' % self.title()) + + wikipedia.get_throttle() + data = query.GetData(params, self.site()) + + for memb in data['query']['categorymembers']: + # For MediaWiki versions where subcats look like articles + if isCatTitle(memb['title'], self.site()): + yield SUBCATEGORY, Category(self.site(), memb['title']) + elif memb['ns'] == 6 and self.site().image_namespace() in memb['title']: + yield ARTICLE, wikipedia.ImagePage(self.site(), title) + else: + yield ARTICLE, wikipedia.Page(self.site(), memb['title']) + # try to find a link to the next list page + if data.has_key('query-continue'): + currentPageOffset = data['query-continue']['categorymembers']['cmcontinue'] + else: + break + + def _oldParseCategory(self, purge=False, startFrom=None): + """ Yields all articles and subcategories that are in this category. Set purge to True to instruct MediaWiki not to serve a cached version. @@ -247,16 +310,14 @@ pass # For MediaWiki versions where subcats look like articles elif isCatTitle(title, self.site()): - ncat = Category(self.site(), title) - yield SUBCATEGORY, ncat + yield SUBCATEGORY, Category(self.site(), title) else: yield ARTICLE, wikipedia.Page(self.site(), title) if Rsubcat: # For MediaWiki versions where subcats look differently for titleWithoutNamespace in Rsubcat.findall(txt): title = 'Category:%s' % titleWithoutNamespace - ncat = Category(self.site(), title) - yield SUBCATEGORY, ncat + yield SUBCATEGORY, Category(self.site(), title) if Rimage: # For MediaWiki versions where images work through galleries for title in Rimage.findall(txt):

1 0

SVN: [7135] trunk/pywikipedia/pagegenerators.py
by alexsh＠svn.wikimedia.org 08 Aug '09

08 Aug '09

Revision: 7135 Author: alexsh Date: 2009-08-08 15:28:21 +0000 (Sat, 08 Aug 2009) Log Message: ----------- pagegenerators.py.UserContributionsGenerator():Change API usage to query.GetData() Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2009-08-08 12:27:09 UTC (rev 7134) +++ trunk/pywikipedia/pagegenerators.py 2009-08-08 15:28:21 UTC (rev 7135) @@ -161,7 +161,7 @@ import urllib, urllib2, time # Application specific imports -import wikipedia, date, catlib +import wikipedia, date, catlib, query import config # For python 2.4 compatibility @@ -488,26 +488,29 @@ if number > 500: # the api does not allow more than 500 results for anonymous users number = 500 - apiQ = site.api_address() + 'action=query&list=usercontribs&ucuser=' - apiQ += urllib.quote(username.encode(site.encoding())) - apiQ += '&ucprop=title&uclimit=%s&format=xml' % number + params = { + 'action': 'query', + 'list': 'usercontribs', + 'ucuser': username, + 'ucprop': 'title', + 'uclimit': int(number), + 'ucdir': 'newer', + } + + if namespaces: - apiQ += '&ucnamespace=%s' % '|'.join(map(str, namespaces)) - titlesRe = re.compile('title="(.*?)"') - ucstartRe = re.compile('ucstart="(.*?)"') - ucstart = '' + params['ucnamespace'] = '|'.join(map(str, namespaces)) # An user is likely to contribute on several pages, # keeping track of titles titleList = [] while True: - result = site.getUrl(apiQ + ucstart) - for title in titlesRe.findall(result): - if not title in titleList: - titleList.append(title) - yield wikipedia.Page(site, title) - m = ucstartRe.search(result) - if m: - ucstart = '&ucstart=' + m.group(1) + result = query.GetData(params, site) + for contr in result['query']['usercontribs']: + if not contr['title'] in titleList: + titleList.append(contr['title']) + yield wikipedia.Page(site, contr['title']) + if result.has_key('query-continue'): + params['ucstart'] = result['query-continue']['usercontribs']['ucstart'] else: break

1 0

SVN: [7134] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 08 Aug '09

08 Aug '09

Revision: 7134 Author: alexsh Date: 2009-08-08 12:27:09 +0000 (Sat, 08 Aug 2009) Log Message: ----------- *Page().delete(), Site().newimages() :Add API NotImplementedError detection. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-08 04:25:51 UTC (rev 7133) +++ trunk/pywikipedia/wikipedia.py 2009-08-08 12:27:09 UTC (rev 7134) @@ -2398,6 +2398,11 @@ if answer == 'y': token = self.site().getToken(self, sysop = True) + try: + d = self.site().api_address() + del d + except NotImplementedError: + config.use_api = False if config.use_api and self.site().versionnumber() >= 12: params = { @@ -2413,10 +2418,11 @@ else: if datas['error']['code'] == 'missingtitle': output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) - return False else: output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) output('%s' % datas) + + return False else: host = self.site().hostname() address = self.site().delete_address(self.urlname()) @@ -5559,11 +5565,15 @@ if leuser is not None: params['leuser'] = leuser if letitle is not None: params['letitle'] = letitle while True: - data = query.GetData(params, self, encodeTitle = False) try: - imagesData = data['query']['logevents'] - except KeyError: + if self.versionnumber() >= 11: + imagesData = query.GetData(params, self, encodeTitle = False)['query']['logevents'] + else: + raise NotImplementedError("The site version is not support this action.") + except KeyError: #no 'query' or 'logevents', error occured raise ServerError("The APIs don't return the data, the site may be down") + except NotImplementedError: + raise ServerError("The site version is not support this action.") for imageData in imagesData: try: @@ -5573,7 +5583,7 @@ pageid = imageData['pageid'] title = imageData['title'] timestamp = imageData['timestamp'] - logid = imageData['logid'] + ##logid = imageData['logid'] #no use current now user = imageData['user'] yield ImagePage(self, title), timestamp, user, comment if not repeat: @@ -5814,6 +5824,7 @@ start = page.titleWithoutNamespace() try: api_url = self.api_address() + del api_url except NotImplementedError: for page in self._allpagesOld(start, namespace, includeredirects, throttle): yield page @@ -5842,8 +5853,9 @@ if data.has_key('query-continue'): start = data['query-continue']['allpages']['apfrom'] + else: + break - def _allpagesOld(self, start='!', namespace=0, includeredirects=True, throttle=True): """

1 0

SVN: [7133] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7133 Author: alexsh Date: 2009-08-08 04:25:51 +0000 (Sat, 08 Aug 2009) Log Message: ----------- Site().newpages(): typo Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-08 04:25:21 UTC (rev 7132) +++ trunk/pywikipedia/wikipedia.py 2009-08-08 04:25:51 UTC (rev 7133) @@ -5327,8 +5327,7 @@ 'rcshow': '!bot|!redirect', #'': '', } - try: - data = query.GetData(params, self)['query']['recentchanges'] + data = query.GetData(params, self)['query']['recentchanges'] for np in data: date = np['timestamp']

1 0

SVN: [7132] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7132 Author: alexsh Date: 2009-08-08 04:25:21 +0000 (Sat, 08 Aug 2009) Log Message: ----------- Site().newpages(): Add API options Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-08 01:00:08 UTC (rev 7131) +++ trunk/pywikipedia/wikipedia.py 2009-08-08 04:25:21 UTC (rev 7132) @@ -5309,30 +5309,62 @@ # option to fetch older rather than newer pages # TODO: extract and return edit comment. seen = set() + try: + d = self.apipath() + del d + except NotImplementedError: + config.use_api = False + while True: - path = self.newpages_address(n=number, namespace=namespace) - # The throttling is important here, so always enabled. - get_throttle() - html = self.getUrl(path) + if config.use_api and self.versionnumber() >= 10: + params = { + 'action': 'query', + 'list': 'recentchanges', + 'rctype': 'new', + 'rcnamespace': namespace, + 'rclimit': int(number), + 'rcprop': 'title|timestamp|sizes|user|comment', + 'rcshow': '!bot|!redirect', + #'': '', + } + try: + data = query.GetData(params, self)['query']['recentchanges'] + + for np in data: + date = np['timestamp'] + title = np['title'] + length = np['newlen'] + username = np['user'] + loggedIn = u'' + comment = np['comment'] + if title not in seen: + seen.add(title) + page = Page(self, title) + yield page, date, length, loggedIn, username, comment + else: + path = self.newpages_address(n=number, namespace=namespace) + # The throttling is important here, so always enabled. + get_throttle() + html = self.getUrl(path) - entryR = re.compile( + entryR = re.compile( '<li[^>]*>(?P<date>.+?) \S*?<a href=".+?"' ' title="(?P<title>.+?)">.+?</a>.+?[$\[](?P<length>[\d,.]+)[^$\]]*[\)\]]' ' .?<a href=".+?" title=".+?:(?P<username>.+?)">' - ) - for m in entryR.finditer(html): - date = m.group('date') - title = m.group('title') - title = title.replace('"', '"') - length = int(re.sub("[,.]", "", m.group('length'))) - loggedIn = u'' - username = m.group('username') - comment = u'' + ) + for m in entryR.finditer(html): + date = m.group('date') + title = m.group('title') + title = title.replace('"', '"') + length = int(re.sub("[,.]", "", m.group('length'))) + loggedIn = u'' + username = m.group('username') + comment = u'' - if title not in seen: - seen.add(title) - page = Page(self, title) - yield page, date, length, loggedIn, username, comment + if title not in seen: + seen.add(title) + page = Page(self, title) + yield page, date, length, loggedIn, username, comment if not repeat: break

1 0

SVN: [7131] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7131 Author: alexsh Date: 2009-08-08 01:00:08 +0000 (Sat, 08 Aug 2009) Log Message: ----------- Change all simplejson.load() datas to use query.getData() Modified Paths: -------------- trunk/pywikipedia/archivebot.py trunk/pywikipedia/category_redirect.py trunk/pywikipedia/commonsdelinker/image_replacer.py trunk/pywikipedia/copyright_clean.py trunk/pywikipedia/revertbot.py Modified: trunk/pywikipedia/archivebot.py =================================================================== --- trunk/pywikipedia/archivebot.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/archivebot.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -34,10 +34,10 @@ # __version__ = '$Id$' # -import wikipedia, pagegenerators +import wikipedia, pagegenerators, query Site = wikipedia.getSite() -import os, re, time, locale, traceback, string, urllib, simplejson +import os, re, time, locale, traceback, string, urllib try: #Get a constructor for the MD5 hash object import hashlib @@ -217,8 +217,7 @@ qdata['eicontinue'] = eicontinue wikipedia.output(u'Fetching template transclusions...') - response, data = Site.postData(Site.apipath(), urllib.urlencode(qdata)) - result = simplejson.loads(data) + response, result = query.GetData(qdata, Site, back_response = True) for page_d in result['query']['embeddedin']: yield wikipedia.Page(Site, page_d['title']) Modified: trunk/pywikipedia/category_redirect.py =================================================================== --- trunk/pywikipedia/category_redirect.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/category_redirect.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -14,9 +14,7 @@ """ __version__ = '$Id$' -import wikipedia, catlib -import pagegenerators -import simplejson +import wikipedia, catlib, query, pagegenerators import cPickle import math import re @@ -302,11 +300,9 @@ def query_results(self, **data): """Iterate results from API action=query, using data as parameters.""" - addr = self.site.apipath() querydata = {'action': 'query', - 'format': 'json', 'maxlag': str(wikipedia.config.maxlag)} - querydata.update(data) + querydata = query.CombineParams(querydata, data) if not querydata.has_key("action")\ or not querydata['action'] == 'query': raise ValueError( @@ -314,19 +310,18 @@ ) waited = 0 while True: - response, data = self.site.postForm(addr, querydata) - if response.status != 200: - # WARNING: if the server is down, this could - # cause an infinite loop - wikipedia.output(u"HTTP error %i received; retrying..." - % response.status) - time.sleep(5) - continue - if data.startswith(u"unknown_action"): - e = {'code': data[:14], 'info': data[16:]} - raise APIError(e) try: - result = simplejson.loads(data) + response, result = query.GetData(predata, self,site back_response = True) + if response.status != 200: + # WARNING: if the server is down, this could + # cause an infinite loop + wikipedia.output(u"HTTP error %i received; retrying..." + % response.status) + time.sleep(5) + continue + if data.startswith(u"unknown_action"): + e = {'code': data[:14], 'info': data[16:]} + raise APIError(e) except ValueError: # if the result isn't valid JSON, there must be a server # problem. Wait a few seconds and try again Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -10,7 +10,7 @@ # Distributed under the terms of the MIT license. # __version__ = '$Id$' -import config, wikipedia, simplejson +import config, wikipedia, query import re, time import threadpool import sys, os, signal, traceback @@ -149,21 +149,19 @@ """ Fetch the last 50 revisions using the API """ address = self.site.api_address() - predata = [ - ('action', 'query'), - ('prop', 'revisions'), - ('titles', title.encode('utf-8')), - ('rvprop', 'timestamp|user|comment|content'), - ('rvlimit', '50'), - ('format', 'json'), - ] + predata = { + 'action': 'query', + 'prop': 'revisions', + 'titles': title.encode('utf-8'), + 'rvprop': 'timestamp|user|comment|content', + 'rvlimit': '50', + } if username: - predata.append(('rvexcludeuser', username.encode('utf-8'))) + predata['rvexcludeuser'] = username.encode('utf-8') if since: - predata.append(('rvend', since)) - response, data = self.site.postForm(address, predata) - data = simplejson.loads(data) - if 'error' in data: + predata['rvend'] = since + response, data = query.GetData(predata, self.site, back_response = True) + if data.has_key('error'): raise RuntimeError(data['error']) page = data['query']['pages'].values()[0] Modified: trunk/pywikipedia/copyright_clean.py =================================================================== --- trunk/pywikipedia/copyright_clean.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/copyright_clean.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -9,7 +9,7 @@ # Distributed under the terms of the MIT license. # -import httplib, socket, simplejson, re, time +import httplib, socket, re, time import config, wikipedia, catlib, pagegenerators, query from urllib import urlencode @@ -33,40 +33,22 @@ rev_templateC = re.compile("(?m)^(?:{{/t\|.*?}}\n?)?{{(?:/box|botbox)\|.*?\|(.*?)\|") def query_api(data): - predata = [ - ('format', 'json'), - ('action', 'query'), - ('prop', 'revisions'), - data] - data = urlencode(predata) - host = wikipedia.getSite().hostname() - address = wikipedia.getSite().api_address() - conn = httplib.HTTPConnection(host) - conn.request("GET", address + data) - response = conn.getresponse() - data = response.read() - conn.close() - return data + predata = { + 'action': 'query', + 'prop': 'revisions', + } + predata = query.CombineParams(predata, data) + return query.GetData(predata) def query_old_api(data): - predata = [ - ('format', 'json'), - ('what', 'revisions'), - ('rvlimit', '1'), - data] + predata = { + 'what': 'revisions', + 'rvlimit': '1', + } + predata = query.CombineParams(predata, data) + return query.GetData(predata, useAPI = False) - data = urlencode(predata) - host = wikipedia.getSite().hostname() - address = wikipedia.getSite().query_address() - conn = httplib.HTTPConnection(host) - conn.request("GET", address + data) - response = conn.getresponse() - data = response.read() - conn.close() - - return data - def old_page_exist(title): for pageobjs in query_results_titles: for key in pageobjs['pages']: @@ -135,9 +117,9 @@ # No more of 50 titles at a time using API for s in mysplit(query.ListToParam(titles), 50, "|"): - query_results_titles.append(simplejson.loads(query_api(('titles', s)))) + query_results_titles.append(query_api({'titles': s,})) for s in mysplit(query.ListToParam(revids), 50, "|"): - query_results_revids.append(simplejson.loads(query_api(('revids', s)))) + query_results_revids.append(query_api({'revids': ,})) comment_entry = list() add_separator = False Modified: trunk/pywikipedia/revertbot.py =================================================================== --- trunk/pywikipedia/revertbot.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/revertbot.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -1,5 +1,4 @@ -import wikipedia -import simplejson +import wikipedia, query __version__ = '$Id$' @@ -18,7 +17,6 @@ self.comment = comment def get_contributions(self, max = -1, ns = None): - address = self.site.api_address() predata = { 'action': 'query', 'list': 'usercontribs', @@ -37,12 +35,11 @@ item = iterator.next() except StopIteration: self.log(u'Fetching new batch of contributions') - response, data = self.site.postForm(address, predata) - data = simplejson.loads(data) - if 'error' in data: + response, data = query.GetData(predata, self.site, back_response = True) + if data.has_key('error'): raise RuntimeError(data['error']) - if 'usercontribs' in data.get('query-continue', ()): - predata.update(data['query-continue']['usercontribs']) + if data.has_key('query-continue'): + predata['uccontinue'] = data['query-continue']['usercontribs'] else: never_continue = True iterator = iter(data['query']['usercontribs']) @@ -81,10 +78,9 @@ 'rvstart': item['timestamp'], 'format': 'json' } - response, data = self.site.postForm(self.site.api_address(), predata) - data = simplejson.loads(data) + response, data = query.GetData(predata, self.site, back_response = True) - if 'error' in data: + if data.has_key('error'): raise RuntimeError(data['error']) pages = data['query'].get('pages', ())

1 0

SVN: [7130] trunk/pywikipedia/simplejson
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7130 Author: alexsh Date: 2009-08-08 00:42:34 +0000 (Sat, 08 Aug 2009) Log Message: ----------- Update simplejson to 2.0.9 Modified Paths: -------------- trunk/pywikipedia/simplejson/__init__.py trunk/pywikipedia/simplejson/_speedups.c trunk/pywikipedia/simplejson/decoder.py trunk/pywikipedia/simplejson/encoder.py trunk/pywikipedia/simplejson/scanner.py Added Paths: ----------- trunk/pywikipedia/simplejson/tool.py Modified: trunk/pywikipedia/simplejson/__init__.py =================================================================== --- trunk/pywikipedia/simplejson/__init__.py 2009-08-07 17:25:10 UTC (rev 7129) +++ trunk/pywikipedia/simplejson/__init__.py 2009-08-08 00:42:34 UTC (rev 7130) @@ -1,97 +1,110 @@ -r""" -A simple, fast, extensible JSON encoder and decoder - -JSON (JavaScript Object Notation) <http://json.org> is a subset of +r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. -simplejson exposes an API familiar to uses of the standard library -marshal and pickle modules. +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. Encoding basic Python object hierarchies:: - - >>> import simplejson - >>> simplejson.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print simplejson.dumps("\"foo\bar") + >>> print json.dumps("\"foo\bar") "\"foo\bar" - >>> print simplejson.dumps(u'\u1234') + >>> print json.dumps(u'\u1234') "\u1234" - >>> print simplejson.dumps('\\') + >>> print json.dumps('\\') "\\" - >>> print simplejson.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) {"a": 0, "b": 0, "c": 0} >>> from StringIO import StringIO >>> io = StringIO() - >>> simplejson.dump(['streaming API'], io) + >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: - >>> import simplejson - >>> simplejson.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: - >>> import simplejson - >>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) { - "4": 5, + "4": 5, "6": 7 } Decoding JSON:: - - >>> import simplejson - >>> simplejson.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') - [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> simplejson.loads('"\\"foo\\bar"') - u'"foo\x08ar' + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True >>> from StringIO import StringIO >>> io = StringIO('["streaming API"]') - >>> simplejson.load(io) - [u'streaming API'] + >>> json.load(io)[0] == 'streaming API' + True Specializing JSON object decoding:: - >>> import simplejson + >>> import simplejson as json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct - ... - >>> simplejson.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) + >>> import decimal + >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1') + True -Extending JSONEncoder:: - - >>> import simplejson - >>> class ComplexEncoder(simplejson.JSONEncoder): - ... def default(self, obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... return simplejson.JSONEncoder.default(self, obj) - ... - >>> dumps(2 + 1j, cls=ComplexEncoder) +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' - >>> ComplexEncoder().encode(2 + 1j) + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' - >>> list(ComplexEncoder().iterencode(2 + 1j)) - ['[', '2.0', ', ', '1.0', ']'] - + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' -Note that the JSON produced by this module's default settings -is a subset of YAML, so it may be used as a serializer for that as well. + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) """ -__version__ = '1.7.3' +__version__ = '2.0.9' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONEncoder', ] +__author__ = 'Bob Ippolito <bob(a)redivi.com>' + from decoder import JSONDecoder from encoder import JSONEncoder @@ -102,31 +115,31 @@ allow_nan=True, indent=None, separators=None, - encoding='utf-8' + encoding='utf-8', + default=None, ) def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', **kw): - """ - Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + encoding='utf-8', default=None, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). - If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is ``False``, then the some chunks written to ``fp`` + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` may be ``unicode`` instances, subject to normal Python ``str`` to ``unicode`` coercion rules. Unless ``fp.write()`` explicitly understands ``unicode`` (as in ``codecs.getwriter()``) this is likely to cause an error. - If ``check_circular`` is ``False``, then the circular reference check + If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). - If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). @@ -141,22 +154,27 @@ ``encoding`` is the character encoding for str instances, default is UTF-8. + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg. + """ # cached encoder - if (skipkeys is False and ensure_ascii is True and - check_circular is True and allow_nan is True and + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and cls is None and indent is None and separators is None and - encoding == 'utf-8' and not kw): + encoding == 'utf-8' and default is None and not kw): iterable = _default_encoder.iterencode(obj) else: if cls is None: cls = JSONEncoder iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, **kw).iterencode(obj) + separators=separators, encoding=encoding, + default=default, **kw).iterencode(obj) # could accelerate with writelines in some versions of Python, at # a debuggability cost for chunk in iterable: @@ -165,23 +183,22 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', **kw): - """ - Serialize ``obj`` to a JSON formatted ``str``. + encoding='utf-8', default=None, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. - If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is ``False``, then the return value will be a + If ``ensure_ascii`` is false, then the return value will be a ``unicode`` instance subject to normal Python ``str`` to ``unicode`` coercion rules instead of being escaped to an ASCII ``str``. - If ``check_circular`` is ``False``, then the circular reference check + If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). - If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). @@ -197,29 +214,35 @@ ``encoding`` is the character encoding for str instances, default is UTF-8. + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg. + """ # cached encoder - if (skipkeys is False and ensure_ascii is True and - check_circular is True and allow_nan is True and + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and cls is None and indent is None and separators is None and - encoding == 'utf-8' and not kw): + encoding == 'utf-8' and default is None and not kw): return _default_encoder.encode(obj) if cls is None: cls = JSONEncoder return cls( skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, + separators=separators, encoding=encoding, default=default, **kw).encode(obj) + _default_decoder = JSONDecoder(encoding=None, object_hook=None) -def load(fp, encoding=None, cls=None, object_hook=None, **kw): - """ - Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. If the contents of ``fp`` is encoded with an ASCII based encoding other @@ -233,16 +256,20 @@ result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg. + """ return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, **kw) + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, **kw) -def loads(s, encoding=None, cls=None, object_hook=None, **kw): - """ - Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) to a Python object. If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding @@ -255,33 +282,37 @@ ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). + ``parse_float``, if specified, will be called with the string + of every JSON float to be decoded. By default this is equivalent to + float(num_str). This can be used to use another datatype or parser + for JSON floats (e.g. decimal.Decimal). + + ``parse_int``, if specified, will be called with the string + of every JSON int to be decoded. By default this is equivalent to + int(num_str). This can be used to use another datatype or parser + for JSON integers (e.g. float). + + ``parse_constant``, if specified, will be called with one of the + following strings: -Infinity, Infinity, NaN, null, true, false. + This can be used to raise an exception if invalid JSON numbers + are encountered. + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg. + """ - if cls is None and encoding is None and object_hook is None and not kw: + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: kw['object_hook'] = object_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant return cls(encoding=encoding, **kw).decode(s) - -def read(s): - """ - json-py API compatibility hook. Use loads(s) instead. - """ - import warnings - warnings.warn("simplejson.loads(s) should be used instead of read(s)", - DeprecationWarning) - return loads(s) - -def write(obj): - """ - json-py API compatibility hook. Use dumps(s) instead. - """ - import warnings - warnings.warn("simplejson.dumps(s) should be used instead of write(s)", - DeprecationWarning) - return dumps(obj) - - Modified: trunk/pywikipedia/simplejson/_speedups.c =================================================================== --- trunk/pywikipedia/simplejson/_speedups.c 2009-08-07 17:25:10 UTC (rev 7129) +++ trunk/pywikipedia/simplejson/_speedups.c 2009-08-08 00:42:34 UTC (rev 7130) @@ -1,10 +1,81 @@ #include "Python.h" +#include "structmember.h" +#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN +#define PyInt_FromSsize_t PyInt_FromLong +#define PyInt_AsSsize_t PyInt_AsLong #endif +#ifndef Py_IS_FINITE +#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) +#endif +#ifdef __GNUC__ +#define UNUSED __attribute__((__unused__)) +#else +#define UNUSED +#endif + +#define DEFAULT_ENCODING "utf-8" + +#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) +#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) +#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) +#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) + +static PyTypeObject PyScannerType; +static PyTypeObject PyEncoderType; + +typedef struct _PyScannerObject { + PyObject_HEAD + PyObject *encoding; + PyObject *strict; + PyObject *object_hook; + PyObject *parse_float; + PyObject *parse_int; + PyObject *parse_constant; +} PyScannerObject; + +static PyMemberDef scanner_members[] = { + {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, + {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"}, + {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, + {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, + {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, + {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, + {NULL} +}; + +typedef struct _PyEncoderObject { + PyObject_HEAD + PyObject *markers; + PyObject *defaultfn; + PyObject *encoder; + PyObject *indent; + PyObject *key_separator; + PyObject *item_separator; + PyObject *sort_keys; + PyObject *skipkeys; + int fast_encode; + int allow_nan; +} PyEncoderObject; + +static PyMemberDef encoder_members[] = { + {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, + {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, + {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, + {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, + {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, + {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, + {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, + {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"}, + {NULL} +}; + static Py_ssize_t ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); static PyObject * @@ -12,10 +83,51 @@ static PyObject * ascii_escape_str(PyObject *pystr); static PyObject * -py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr); +py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); void init_speedups(void); +static PyObject * +scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); +static PyObject * +scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); +static PyObject * +_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); +static PyObject * +scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int +scanner_init(PyObject *self, PyObject *args, PyObject *kwds); +static void +scanner_dealloc(PyObject *self); +static int +scanner_clear(PyObject *self); +static PyObject * +encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int +encoder_init(PyObject *self, PyObject *args, PyObject *kwds); +static void +encoder_dealloc(PyObject *self); +static int +encoder_clear(PyObject *self); +static int +encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); +static int +encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); +static int +encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); +static PyObject * +_encoded_const(PyObject *const); +static void +raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); +static PyObject * +encoder_encode_string(PyEncoderObject *s, PyObject *obj); +static int +_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); +static PyObject * +_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); +static PyObject * +encoder_encode_float(PyEncoderObject *s, PyObject *obj); -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '/' && c != '"') +#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') +#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) #define MIN_EXPANSION 6 #ifdef Py_UNICODE_WIDE @@ -24,12 +136,31 @@ #define MAX_EXPANSION MIN_EXPANSION #endif +static int +_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) +{ + /* PyObject to Py_ssize_t converter */ + *size_ptr = PyInt_AsSsize_t(o); + if (*size_ptr == -1 && PyErr_Occurred()); + return 1; + return 0; +} + +static PyObject * +_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) +{ + /* Py_ssize_t to PyObject converter */ + return PyInt_FromSsize_t(*size_ptr); +} + static Py_ssize_t -ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) { - Py_UNICODE x; +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) +{ + /* Escape unicode code point c to ASCII escape sequences + in char *output. output must have at least 12 bytes unused to + accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ output[chars++] = '\\'; switch (c) { - case '/': output[chars++] = (char)c; break; case '\\': output[chars++] = (char)c; break; case '"': output[chars++] = (char)c; break; case '\b': output[chars++] = 'b'; break; @@ -44,36 +175,31 @@ Py_UNICODE v = c - 0x10000; c = 0xd800 | ((v >> 10) & 0x3ff); output[chars++] = 'u'; - x = (c & 0xf000) >> 12; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x0f00) >> 8; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x00f0) >> 4; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x000f); - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; + output[chars++] = "0123456789abcdef"[(c ) & 0xf]; c = 0xdc00 | (v & 0x3ff); output[chars++] = '\\'; } #endif output[chars++] = 'u'; - x = (c & 0xf000) >> 12; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x0f00) >> 8; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x00f0) >> 4; - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); - x = (c & 0x000f); - output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; + output[chars++] = "0123456789abcdef"[(c ) & 0xf]; } return chars; } static PyObject * -ascii_escape_unicode(PyObject *pystr) { +ascii_escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ Py_ssize_t i; Py_ssize_t input_chars; Py_ssize_t output_size; + Py_ssize_t max_output_size; Py_ssize_t chars; PyObject *rval; char *output; @@ -81,8 +207,10 @@ input_chars = PyUnicode_GET_SIZE(pystr); input_unicode = PyUnicode_AS_UNICODE(pystr); + /* One char input can be up to 6 chars output, estimate 4 of these */ output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + max_output_size = 2 + (input_chars * MAX_EXPANSION); rval = PyString_FromStringAndSize(NULL, output_size); if (rval == NULL) { return NULL; @@ -94,20 +222,25 @@ Py_UNICODE c = input_unicode[i]; if (S_CHAR(c)) { output[chars++] = (char)c; - } else { + } + else { chars = ascii_escape_char(c, output, chars); } if (output_size - chars < (1 + MAX_EXPANSION)) { /* There's more than four, so let's resize by a lot */ - output_size *= 2; + Py_ssize_t new_output_size = output_size * 2; /* This is an upper bound */ - if (output_size > 2 + (input_chars * MAX_EXPANSION)) { - output_size = 2 + (input_chars * MAX_EXPANSION); + if (new_output_size > max_output_size) { + new_output_size = max_output_size; } - if (_PyString_Resize(&rval, output_size) == -1) { - return NULL; + /* Make sure that the output size changed before resizing */ + if (new_output_size != output_size) { + output_size = new_output_size; + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); } - output = PyString_AS_STRING(rval); } } output[chars++] = '"'; @@ -118,7 +251,9 @@ } static PyObject * -ascii_escape_str(PyObject *pystr) { +ascii_escape_str(PyObject *pystr) +{ + /* Take a PyString pystr and return a new ASCII-only escaped PyString */ Py_ssize_t i; Py_ssize_t input_chars; Py_ssize_t output_size; @@ -129,31 +264,56 @@ input_chars = PyString_GET_SIZE(pystr); input_str = PyString_AS_STRING(pystr); - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + + /* Fast path for a string that's already ASCII */ + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; + if (!S_CHAR(c)) { + /* If we have to escape something, scan the string for unicode */ + Py_ssize_t j; + for (j = i; j < input_chars; j++) { + c = (Py_UNICODE)(unsigned char)input_str[j]; + if (c > 0x7f) { + /* We hit a non-ASCII character, bail to unicode mode */ + PyObject *uni; + uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); + if (uni == NULL) { + return NULL; + } + rval = ascii_escape_unicode(uni); + Py_DECREF(uni); + return rval; + } + } + break; + } + } + + if (i == input_chars) { + /* Input is already ASCII */ + output_size = 2 + input_chars; + } + else { + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + } rval = PyString_FromStringAndSize(NULL, output_size); if (rval == NULL) { return NULL; } output = PyString_AS_STRING(rval); - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - Py_UNICODE c = (Py_UNICODE)input_str[i]; + output[0] = '"'; + + /* We know that everything up to i is ASCII already */ + chars = i + 1; + memcpy(&output[1], input_str, i); + + for (; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; if (S_CHAR(c)) { output[chars++] = (char)c; - } else if (c > 0x7F) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - Py_DECREF(rval); - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } else { + } + else { chars = ascii_escape_char(c, output, chars); } /* An ASCII char can't possibly expand to a surrogate! */ @@ -176,40 +336,1994 @@ return rval; } +static void +raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) +{ + /* Use the Python function simplejson.decoder.errmsg to raise a nice + looking ValueError exception */ + static PyObject *errmsg_fn = NULL; + PyObject *pymsg; + if (errmsg_fn == NULL) { + PyObject *decoder = PyImport_ImportModule("simplejson.decoder"); + if (decoder == NULL) + return; + errmsg_fn = PyObject_GetAttrString(decoder, "errmsg"); + Py_DECREF(decoder); + if (errmsg_fn == NULL) + return; + } + pymsg = PyObject_CallFunction(errmsg_fn, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); + if (pymsg) { + PyErr_SetObject(PyExc_ValueError, pymsg); + Py_DECREF(pymsg); + } +} + +static PyObject * +join_list_unicode(PyObject *lst) +{ + /* return u''.join(lst) */ + static PyObject *joinfn = NULL; + if (joinfn == NULL) { + PyObject *ustr = PyUnicode_FromUnicode(NULL, 0); + if (ustr == NULL) + return NULL; + + joinfn = PyObject_GetAttrString(ustr, "join"); + Py_DECREF(ustr); + if (joinfn == NULL) + return NULL; + } + return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); +} + +static PyObject * +join_list_string(PyObject *lst) +{ + /* return ''.join(lst) */ + static PyObject *joinfn = NULL; + if (joinfn == NULL) { + PyObject *ustr = PyString_FromStringAndSize(NULL, 0); + if (ustr == NULL) + return NULL; + + joinfn = PyObject_GetAttrString(ustr, "join"); + Py_DECREF(ustr); + if (joinfn == NULL) + return NULL; + } + return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); +} + +static PyObject * +_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { + /* return (rval, idx) tuple, stealing reference to rval */ + PyObject *tpl; + PyObject *pyidx; + /* + steal a reference to rval, returns (rval, idx) + */ + if (rval == NULL) { + return NULL; + } + pyidx = PyInt_FromSsize_t(idx); + if (pyidx == NULL) { + Py_DECREF(rval); + return NULL; + } + tpl = PyTuple_New(2); + if (tpl == NULL) { + Py_DECREF(pyidx); + Py_DECREF(rval); + return NULL; + } + PyTuple_SET_ITEM(tpl, 0, rval); + PyTuple_SET_ITEM(tpl, 1, pyidx); + return tpl; +} + +static PyObject * +scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) +{ + /* Read the JSON string from PyString pystr. + end is the index of the first character after the quote. + encoding is the encoding of pystr (must be an ASCII superset) + if strict is zero then literal control characters are allowed + *next_end_ptr is a return-by-reference index of the character + after the end quote + + Return value is a new PyString (if ASCII-only) or PyUnicode + */ + PyObject *rval; + Py_ssize_t len = PyString_GET_SIZE(pystr); + Py_ssize_t begin = end - 1; + Py_ssize_t next = begin; + int has_unicode = 0; + char *buf = PyString_AS_STRING(pystr); + PyObject *chunks = PyList_New(0); + if (chunks == NULL) { + goto bail; + } + if (end < 0 || len <= end) { + PyErr_SetString(PyExc_ValueError, "end is out of bounds"); + goto bail; + } + while (1) { + /* Find the end of the string or the next escape */ + Py_UNICODE c = 0; + PyObject *chunk = NULL; + for (next = end; next < len; next++) { + c = (unsigned char)buf[next]; + if (c == '"' || c == '\\') { + break; + } + else if (strict && c <= 0x1f) { + raise_errmsg("Invalid control character at", pystr, next); + goto bail; + } + else if (c > 0x7f) { + has_unicode = 1; + } + } + if (!(c == '"' || c == '\\')) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + /* Pick up this chunk if it's not zero length */ + if (next != end) { + PyObject *strchunk = PyString_FromStringAndSize(&buf[end], next - end); + if (strchunk == NULL) { + goto bail; + } + if (has_unicode) { + chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); + Py_DECREF(strchunk); + if (chunk == NULL) { + goto bail; + } + } + else { + chunk = strchunk; + } + if (PyList_Append(chunks, chunk)) { + Py_DECREF(chunk); + goto bail; + } + Py_DECREF(chunk); + } + next++; + if (c == '"') { + end = next; + break; + } + if (next == len) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + c = buf[next]; + if (c != 'u') { + /* Non-unicode backslash escapes */ + end = next + 1; + switch (c) { + case '"': break; + case '\\': break; + case '/': break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + default: c = 0; + } + if (c == 0) { + raise_errmsg("Invalid \\escape", pystr, end - 2); + goto bail; + } + } + else { + c = 0; + next++; + end = next + 4; + if (end >= len) { + raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); + goto bail; + } + /* Decode 4 hex digits */ + for (; next < end; next++) { + Py_UNICODE digit = buf[next]; + c <<= 4; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } +#ifdef Py_UNICODE_WIDE + /* Surrogate pair */ + if ((c & 0xfc00) == 0xd800) { + Py_UNICODE c2 = 0; + if (end + 6 >= len) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + if (buf[next++] != '\\' || buf[next++] != 'u') { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + end += 6; + /* Decode 4 hex digits */ + for (; next < end; next++) { + c2 <<= 4; + Py_UNICODE digit = buf[next]; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c2 |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c2 |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c2 |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } + if ((c2 & 0xfc00) != 0xdc00) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); + } + else if ((c & 0xfc00) == 0xdc00) { + raise_errmsg("Unpaired low surrogate", pystr, end - 5); + goto bail; + } +#endif + } + if (c > 0x7f) { + has_unicode = 1; + } + if (has_unicode) { + chunk = PyUnicode_FromUnicode(&c, 1); + if (chunk == NULL) { + goto bail; + } + } + else { + char c_char = Py_CHARMASK(c); + chunk = PyString_FromStringAndSize(&c_char, 1); + if (chunk == NULL) { + goto bail; + } + } + if (PyList_Append(chunks, chunk)) { + Py_DECREF(chunk); + goto bail; + } + Py_DECREF(chunk); + } + + rval = join_list_string(chunks); + if (rval == NULL) { + goto bail; + } + Py_CLEAR(chunks); + *next_end_ptr = end; + return rval; +bail: + *next_end_ptr = -1; + Py_XDECREF(chunks); + return NULL; +} + + +static PyObject * +scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) +{ + /* Read the JSON string from PyUnicode pystr. + end is the index of the first character after the quote. + if strict is zero then literal control characters are allowed + *next_end_ptr is a return-by-reference index of the character + after the end quote + + Return value is a new PyUnicode + */ + PyObject *rval; + Py_ssize_t len = PyUnicode_GET_SIZE(pystr); + Py_ssize_t begin = end - 1; + Py_ssize_t next = begin; + const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); + PyObject *chunks = PyList_New(0); + if (chunks == NULL) { + goto bail; + } + if (end < 0 || len <= end) { + PyErr_SetString(PyExc_ValueError, "end is out of bounds"); + goto bail; + } + while (1) { + /* Find the end of the string or the next escape */ + Py_UNICODE c = 0; + PyObject *chunk = NULL; + for (next = end; next < len; next++) { + c = buf[next]; + if (c == '"' || c == '\\') { + break; + } + else if (strict && c <= 0x1f) { + raise_errmsg("Invalid control character at", pystr, next); + goto bail; + } + } + if (!(c == '"' || c == '\\')) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + /* Pick up this chunk if it's not zero length */ + if (next != end) { + chunk = PyUnicode_FromUnicode(&buf[end], next - end); + if (chunk == NULL) { + goto bail; + } + if (PyList_Append(chunks, chunk)) { + Py_DECREF(chunk); + goto bail; + } + Py_DECREF(chunk); + } + next++; + if (c == '"') { + end = next; + break; + } + if (next == len) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + c = buf[next]; + if (c != 'u') { + /* Non-unicode backslash escapes */ + end = next + 1; + switch (c) { + case '"': break; + case '\\': break; + case '/': break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + default: c = 0; + } + if (c == 0) { + raise_errmsg("Invalid \\escape", pystr, end - 2); + goto bail; + } + } + else { + c = 0; + next++; + end = next + 4; + if (end >= len) { + raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); + goto bail; + } + /* Decode 4 hex digits */ + for (; next < end; next++) { + Py_UNICODE digit = buf[next]; + c <<= 4; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } +#ifdef Py_UNICODE_WIDE + /* Surrogate pair */ + if ((c & 0xfc00) == 0xd800) { + Py_UNICODE c2 = 0; + if (end + 6 >= len) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + if (buf[next++] != '\\' || buf[next++] != 'u') { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + end += 6; + /* Decode 4 hex digits */ + for (; next < end; next++) { + c2 <<= 4; + Py_UNICODE digit = buf[next]; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c2 |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c2 |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c2 |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } + if ((c2 & 0xfc00) != 0xdc00) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); + } + else if ((c & 0xfc00) == 0xdc00) { + raise_errmsg("Unpaired low surrogate", pystr, end - 5); + goto bail; + } +#endif + } + chunk = PyUnicode_FromUnicode(&c, 1); + if (chunk == NULL) { + goto bail; + } + if (PyList_Append(chunks, chunk)) { + Py_DECREF(chunk); + goto bail; + } + Py_DECREF(chunk); + } + + rval = join_list_unicode(chunks); + if (rval == NULL) { + goto bail; + } + Py_DECREF(chunks); + *next_end_ptr = end; + return rval; +bail: + *next_end_ptr = -1; + Py_XDECREF(chunks); + return NULL; +} + +PyDoc_STRVAR(pydoc_scanstring, + "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" + "\n" + "Scan the string s for a JSON string. End is the index of the\n" + "character in s after the quote that started the JSON string.\n" + "Unescapes all valid JSON string escape sequences and raises ValueError\n" + "on attempt to decode an invalid string. If strict is False then literal\n" + "control characters are allowed in the string.\n" + "\n" + "Returns a tuple of the decoded string and the index of the character in s\n" + "after the end quote." +); + +static PyObject * +py_scanstring(PyObject* self UNUSED, PyObject *args) +{ + PyObject *pystr; + PyObject *rval; + Py_ssize_t end; + Py_ssize_t next_end = -1; + char *encoding = NULL; + int strict = 1; + if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { + return NULL; + } + if (encoding == NULL) { + encoding = DEFAULT_ENCODING; + } + if (PyString_Check(pystr)) { + rval = scanstring_str(pystr, end, encoding, strict, &next_end); + } + else if (PyUnicode_Check(pystr)) { + rval = scanstring_unicode(pystr, end, strict, &next_end); + } + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } + return _build_rval_index_tuple(rval, next_end); +} + PyDoc_STRVAR(pydoc_encode_basestring_ascii, "encode_basestring_ascii(basestring) -> str\n" "\n" - "..." + "Return an ASCII-only JSON representation of a Python string" ); static PyObject * -py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr) { +py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) +{ + /* Return an ASCII-only JSON representation of a Python string */ /* METH_O */ if (PyString_Check(pystr)) { return ascii_escape_str(pystr); - } else if (PyUnicode_Check(pystr)) { + } + else if (PyUnicode_Check(pystr)) { return ascii_escape_unicode(pystr); } - PyErr_SetString(PyExc_TypeError, "first argument must be a string"); + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } +} + +static void +scanner_dealloc(PyObject *self) +{ + /* Deallocate scanner object */ + scanner_clear(self); + Py_TYPE(self)->tp_free(self); +} + +static int +scanner_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + Py_VISIT(s->encoding); + Py_VISIT(s->strict); + Py_VISIT(s->object_hook); + Py_VISIT(s->parse_float); + Py_VISIT(s->parse_int); + Py_VISIT(s->parse_constant); + return 0; +} + +static int +scanner_clear(PyObject *self) +{ + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + Py_CLEAR(s->encoding); + Py_CLEAR(s->strict); + Py_CLEAR(s->object_hook); + Py_CLEAR(s->parse_float); + Py_CLEAR(s->parse_int); + Py_CLEAR(s->parse_constant); + return 0; +} + +static PyObject * +_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON object from PyString pystr. + idx is the index of the first character after the opening curly brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing curly brace. + + Returns a new PyObject (usually a dict, but object_hook can change that) + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + PyObject *rval = PyDict_New(); + PyObject *key = NULL; + PyObject *val = NULL; + char *encoding = PyString_AS_STRING(s->encoding); + int strict = PyObject_IsTrue(s->strict); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after { */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the object is non-empty */ + if (idx <= end_idx && str[idx] != '}') { + while (idx <= end_idx) { + /* read key */ + if (str[idx] != '"') { + raise_errmsg("Expecting property name", pystr, idx); + goto bail; + } + key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); + if (key == NULL) + goto bail; + idx = next_idx; + + /* skip whitespace between key and : delimiter, read :, skip whitespace */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + if (idx > end_idx || str[idx] != ':') { + raise_errmsg("Expecting : delimiter", pystr, idx); + goto bail; + } + idx++; + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* read any JSON data type */ + val = scan_once_str(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (PyDict_SetItem(rval, key, val) == -1) + goto bail; + + Py_CLEAR(key); + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace before } or , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the object is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == '}') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , delimiter */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + /* verify that idx < end_idx, str[idx] should be '}' */ + if (idx > end_idx || str[idx] != '}') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + /* if object_hook is not None: rval = object_hook(rval) */ + if (s->object_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + if (val == NULL) + goto bail; + Py_DECREF(rval); + rval = val; + val = NULL; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(key); + Py_XDECREF(val); + Py_DECREF(rval); return NULL; } -#define DEFN(n, k) \ - { \ - #n, \ - (PyCFunction)py_ ##n, \ - k, \ - pydoc_ ##n \ +static PyObject * +_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON object from PyUnicode pystr. + idx is the index of the first character after the opening curly brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing curly brace. + + Returns a new PyObject (usually a dict, but object_hook can change that) + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + PyObject *val = NULL; + PyObject *rval = PyDict_New(); + PyObject *key = NULL; + int strict = PyObject_IsTrue(s->strict); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after { */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the object is non-empty */ + if (idx <= end_idx && str[idx] != '}') { + while (idx <= end_idx) { + /* read key */ + if (str[idx] != '"') { + raise_errmsg("Expecting property name", pystr, idx); + goto bail; + } + key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); + if (key == NULL) + goto bail; + idx = next_idx; + + /* skip whitespace between key and : delimiter, read :, skip whitespace */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + if (idx > end_idx || str[idx] != ':') { + raise_errmsg("Expecting : delimiter", pystr, idx); + goto bail; + } + idx++; + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* read any JSON term */ + val = scan_once_unicode(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (PyDict_SetItem(rval, key, val) == -1) + goto bail; + + Py_CLEAR(key); + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace before } or , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the object is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == '}') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , delimiter */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } } + + /* verify that idx < end_idx, str[idx] should be '}' */ + if (idx > end_idx || str[idx] != '}') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + + /* if object_hook is not None: rval = object_hook(rval) */ + if (s->object_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + if (val == NULL) + goto bail; + Py_DECREF(rval); + rval = val; + val = NULL; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(key); + Py_XDECREF(val); + Py_DECREF(rval); + return NULL; +} + +static PyObject * +_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON array from PyString pystr. + idx is the index of the first character after the opening brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing brace. + + Returns a new PyList + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + PyObject *val = NULL; + PyObject *rval = PyList_New(0); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after [ */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the array is non-empty */ + if (idx <= end_idx && str[idx] != ']') { + while (idx <= end_idx) { + + /* read any JSON term and de-tuplefy the (rval, idx) */ + val = scan_once_str(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (PyList_Append(rval, val) == -1) + goto bail; + + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace between term and , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the array is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == ']') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + + /* verify that idx < end_idx, str[idx] should be ']' */ + if (idx > end_idx || str[idx] != ']') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(val); + Py_DECREF(rval); + return NULL; +} + +static PyObject * +_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON array from PyString pystr. + idx is the index of the first character after the opening brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing brace. + + Returns a new PyList + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + PyObject *val = NULL; + PyObject *rval = PyList_New(0); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after [ */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the array is non-empty */ + if (idx <= end_idx && str[idx] != ']') { + while (idx <= end_idx) { + + /* read any JSON term */ + val = scan_once_unicode(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (PyList_Append(rval, val) == -1) + goto bail; + + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace between term and , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the array is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == ']') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + + /* verify that idx < end_idx, str[idx] should be ']' */ + if (idx > end_idx || str[idx] != ']') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(val); + Py_DECREF(rval); + return NULL; +} + +static PyObject * +_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON constant from PyString pystr. + constant is the constant string that was found + ("NaN", "Infinity", "-Infinity"). + idx is the index of the first character of the constant + *next_idx_ptr is a return-by-reference index to the first character after + the constant. + + Returns the result of parse_constant + */ + PyObject *cstr; + PyObject *rval; + /* constant is "NaN", "Infinity", or "-Infinity" */ + cstr = PyString_InternFromString(constant); + if (cstr == NULL) + return NULL; + + /* rval = parse_constant(constant) */ + rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); + idx += PyString_GET_SIZE(cstr); + Py_DECREF(cstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { + /* Read a JSON number from PyString pystr. + idx is the index of the first character of the number + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of that number: + PyInt, PyLong, or PyFloat. + May return other types if parse_int or parse_float are set + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + Py_ssize_t idx = start; + int is_float = 0; + PyObject *rval; + PyObject *numstr; + + /* read a sign if it's there, make sure it's not the end of the string */ + if (str[idx] == '-') { + idx++; + if (idx > end_idx) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + } + + /* read as many integer digits as we find as long as it doesn't start with 0 */ + if (str[idx] >= '1' && str[idx] <= '9') { + idx++; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + /* if it starts with 0 we only expect one integer digit */ + else if (str[idx] == '0') { + idx++; + } + /* no integer digits, error */ + else { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + /* if the next char is '.' followed by a digit then read all float digits */ + if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + is_float = 1; + idx += 2; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + + /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ + if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + + /* save the index of the 'e' or 'E' just in case we need to backtrack */ + Py_ssize_t e_start = idx; + idx++; + + /* read an exponent sign if present */ + if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + + /* read all digits */ + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + + /* if we got a digit, then parse as float. if not, backtrack */ + if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + is_float = 1; + } + else { + idx = e_start; + } + } + + /* copy the section we determined to be a number */ + numstr = PyString_FromStringAndSize(&str[start], idx - start); + if (numstr == NULL) + return NULL; + if (is_float) { + /* parse as a float using a fast path if available, otherwise call user defined method */ + if (s->parse_float != (PyObject *)&PyFloat_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + } + else { + rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); + } + } + else { + /* parse as an int using a fast path if available, otherwise call user defined method */ + if (s->parse_int != (PyObject *)&PyInt_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + } + else { + rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); + } + } + Py_DECREF(numstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { + /* Read a JSON number from PyUnicode pystr. + idx is the index of the first character of the number + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of that number: + PyInt, PyLong, or PyFloat. + May return other types if parse_int or parse_float are set + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + Py_ssize_t idx = start; + int is_float = 0; + PyObject *rval; + PyObject *numstr; + + /* read a sign if it's there, make sure it's not the end of the string */ + if (str[idx] == '-') { + idx++; + if (idx > end_idx) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + } + + /* read as many integer digits as we find as long as it doesn't start with 0 */ + if (str[idx] >= '1' && str[idx] <= '9') { + idx++; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + /* if it starts with 0 we only expect one integer digit */ + else if (str[idx] == '0') { + idx++; + } + /* no integer digits, error */ + else { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + /* if the next char is '.' followed by a digit then read all float digits */ + if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + is_float = 1; + idx += 2; + while (idx < end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + + /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ + if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + Py_ssize_t e_start = idx; + idx++; + + /* read an exponent sign if present */ + if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + + /* read all digits */ + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + + /* if we got a digit, then parse as float. if not, backtrack */ + if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + is_float = 1; + } + else { + idx = e_start; + } + } + + /* copy the section we determined to be a number */ + numstr = PyUnicode_FromUnicode(&str[start], idx - start); + if (numstr == NULL) + return NULL; + if (is_float) { + /* parse as a float using a fast path if available, otherwise call user defined method */ + if (s->parse_float != (PyObject *)&PyFloat_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + } + else { + rval = PyFloat_FromString(numstr, NULL); + } + } + else { + /* no fast path for unicode -> int, just call */ + rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + } + Py_DECREF(numstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +{ + /* Read one JSON term (of any kind) from PyString pystr. + idx is the index of the first character of the term + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of the term. + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t length = PyString_GET_SIZE(pystr); + if (idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + switch (str[idx]) { + case '"': + /* string */ + return scanstring_str(pystr, idx + 1, + PyString_AS_STRING(s->encoding), + PyObject_IsTrue(s->strict), + next_idx_ptr); + case '{': + /* object */ + return _parse_object_str(s, pystr, idx + 1, next_idx_ptr); + case '[': + /* array */ + return _parse_array_str(s, pystr, idx + 1, next_idx_ptr); + case 'n': + /* null */ + if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + Py_INCREF(Py_None); + *next_idx_ptr = idx + 4; + return Py_None; + } + break; + case 't': + /* true */ + if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + Py_INCREF(Py_True); + *next_idx_ptr = idx + 4; + return Py_True; + } + break; + case 'f': + /* false */ + if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + Py_INCREF(Py_False); + *next_idx_ptr = idx + 5; + return Py_False; + } + break; + case 'N': + /* NaN */ + if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + return _parse_constant(s, "NaN", idx, next_idx_ptr); + } + break; + case 'I': + /* Infinity */ + if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + return _parse_constant(s, "Infinity", idx, next_idx_ptr); + } + break; + case '-': + /* -Infinity */ + if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + return _parse_constant(s, "-Infinity", idx, next_idx_ptr); + } + break; + } + /* Didn't find a string, object, array, or named constant. Look for a number. */ + return _match_number_str(s, pystr, idx, next_idx_ptr); +} + +static PyObject * +scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +{ + /* Read one JSON term (of any kind) from PyUnicode pystr. + idx is the index of the first character of the term + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of the term. + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t length = PyUnicode_GET_SIZE(pystr); + if (idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + switch (str[idx]) { + case '"': + /* string */ + return scanstring_unicode(pystr, idx + 1, + PyObject_IsTrue(s->strict), + next_idx_ptr); + case '{': + /* object */ + return _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); + case '[': + /* array */ + return _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); + case 'n': + /* null */ + if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + Py_INCREF(Py_None); + *next_idx_ptr = idx + 4; + return Py_None; + } + break; + case 't': + /* true */ + if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + Py_INCREF(Py_True); + *next_idx_ptr = idx + 4; + return Py_True; + } + break; + case 'f': + /* false */ + if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + Py_INCREF(Py_False); + *next_idx_ptr = idx + 5; + return Py_False; + } + break; + case 'N': + /* NaN */ + if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + return _parse_constant(s, "NaN", idx, next_idx_ptr); + } + break; + case 'I': + /* Infinity */ + if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + return _parse_constant(s, "Infinity", idx, next_idx_ptr); + } + break; + case '-': + /* -Infinity */ + if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + return _parse_constant(s, "-Infinity", idx, next_idx_ptr); + } + break; + } + /* Didn't find a string, object, array, or named constant. Look for a number. */ + return _match_number_unicode(s, pystr, idx, next_idx_ptr); +} + +static PyObject * +scanner_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Python callable interface to scan_once_{str,unicode} */ + PyObject *pystr; + PyObject *rval; + Py_ssize_t idx; + Py_ssize_t next_idx = -1; + static char *kwlist[] = {"string", "idx", NULL}; + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) + return NULL; + + if (PyString_Check(pystr)) { + rval = scan_once_str(s, pystr, idx, &next_idx); + } + else if (PyUnicode_Check(pystr)) { + rval = scan_once_unicode(s, pystr, idx, &next_idx); + } + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } + return _build_rval_index_tuple(rval, next_idx); +} + +static PyObject * +scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyScannerObject *s; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s != NULL) { + s->encoding = NULL; + s->strict = NULL; + s->object_hook = NULL; + s->parse_float = NULL; + s->parse_int = NULL; + s->parse_constant = NULL; + } + return (PyObject *)s; +} + +static int +scanner_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Initialize Scanner object */ + PyObject *ctx; + static char *kwlist[] = {"context", NULL}; + PyScannerObject *s; + + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) + return -1; + + /* PyString_AS_STRING is used on encoding */ + s->encoding = PyObject_GetAttrString(ctx, "encoding"); + if (s->encoding == Py_None) { + Py_DECREF(Py_None); + s->encoding = PyString_InternFromString(DEFAULT_ENCODING); + } + else if (PyUnicode_Check(s->encoding)) { + PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL); + Py_DECREF(s->encoding); + s->encoding = tmp; + } + if (s->encoding == NULL || !PyString_Check(s->encoding)) + goto bail; + + /* All of these will fail "gracefully" so we don't need to verify them */ + s->strict = PyObject_GetAttrString(ctx, "strict"); + if (s->strict == NULL) + goto bail; + s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); + if (s->object_hook == NULL) + goto bail; + s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); + if (s->parse_float == NULL) + goto bail; + s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); + if (s->parse_int == NULL) + goto bail; + s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); + if (s->parse_constant == NULL) + goto bail; + + return 0; + +bail: + Py_CLEAR(s->encoding); + Py_CLEAR(s->strict); + Py_CLEAR(s->object_hook); + Py_CLEAR(s->parse_float); + Py_CLEAR(s->parse_int); + Py_CLEAR(s->parse_constant); + return -1; +} + +PyDoc_STRVAR(scanner_doc, "JSON scanner object"); + +static +PyTypeObject PyScannerType = { + PyObject_HEAD_INIT(NULL) + 0, /* tp_internal */ + "simplejson._speedups.Scanner", /* tp_name */ + sizeof(PyScannerObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + scanner_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + scanner_call, /* tp_call */ + 0, /* tp_str */ + 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ + 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + scanner_doc, /* tp_doc */ + scanner_traverse, /* tp_traverse */ + scanner_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + scanner_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + scanner_init, /* tp_init */ + 0,/* PyType_GenericAlloc, */ /* tp_alloc */ + scanner_new, /* tp_new */ + 0,/* PyObject_GC_Del, */ /* tp_free */ +}; + +static PyObject * +encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyEncoderObject *s; + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s != NULL) { + s->markers = NULL; + s->defaultfn = NULL; + s->encoder = NULL; + s->indent = NULL; + s->key_separator = NULL; + s->item_separator = NULL; + s->sort_keys = NULL; + s->skipkeys = NULL; + } + return (PyObject *)s; +} + +static int +encoder_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* initialize Encoder object */ + static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; + + PyEncoderObject *s; + PyObject *allow_nan; + + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOO:make_encoder", kwlist, + &s->markers, &s->defaultfn, &s->encoder, &s->indent, &s->key_separator, &s->item_separator, &s->sort_keys, &s->skipkeys, &allow_nan)) + return -1; + + Py_INCREF(s->markers); + Py_INCREF(s->defaultfn); + Py_INCREF(s->encoder); + Py_INCREF(s->indent); + Py_INCREF(s->key_separator); + Py_INCREF(s->item_separator); + Py_INCREF(s->sort_keys); + Py_INCREF(s->skipkeys); + s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); + s->allow_nan = PyObject_IsTrue(allow_nan); + return 0; +} + +static PyObject * +encoder_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Python callable interface to encode_listencode_obj */ + static char *kwlist[] = {"obj", "_current_indent_level", NULL}; + PyObject *obj; + PyObject *rval; + Py_ssize_t indent_level; + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, + &obj, _convertPyInt_AsSsize_t, &indent_level)) + return NULL; + rval = PyList_New(0); + if (rval == NULL) + return NULL; + if (encoder_listencode_obj(s, rval, obj, indent_level)) { + Py_DECREF(rval); + return NULL; + } + return rval; +} + +static PyObject * +_encoded_const(PyObject *obj) +{ + /* Return the JSON string representation of None, True, False */ + if (obj == Py_None) { + static PyObject *s_null = NULL; + if (s_null == NULL) { + s_null = PyString_InternFromString("null"); + } + Py_INCREF(s_null); + return s_null; + } + else if (obj == Py_True) { + static PyObject *s_true = NULL; + if (s_true == NULL) { + s_true = PyString_InternFromString("true"); + } + Py_INCREF(s_true); + return s_true; + } + else if (obj == Py_False) { + static PyObject *s_false = NULL; + if (s_false == NULL) { + s_false = PyString_InternFromString("false"); + } + Py_INCREF(s_false); + return s_false; + } + else { + PyErr_SetString(PyExc_ValueError, "not a const"); + return NULL; + } +} + +static PyObject * +encoder_encode_float(PyEncoderObject *s, PyObject *obj) +{ + /* Return the JSON representation of a PyFloat */ + double i = PyFloat_AS_DOUBLE(obj); + if (!Py_IS_FINITE(i)) { + if (!s->allow_nan) { + PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); + return NULL; + } + if (i > 0) { + return PyString_FromString("Infinity"); + } + else if (i < 0) { + return PyString_FromString("-Infinity"); + } + else { + return PyString_FromString("NaN"); + } + } + /* Use a better float format here? */ + return PyObject_Repr(obj); +} + +static PyObject * +encoder_encode_string(PyEncoderObject *s, PyObject *obj) +{ + /* Return the JSON representation of a string */ + if (s->fast_encode) + return py_encode_basestring_ascii(NULL, obj); + else + return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); +} + +static int +_steal_list_append(PyObject *lst, PyObject *stolen) +{ + /* Append stolen and then decrement its reference count */ + int rval = PyList_Append(lst, stolen); + Py_DECREF(stolen); + return rval; +} + +static int +encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) +{ + /* Encode Python object obj to a JSON term, rval is a PyList */ + PyObject *newobj; + int rv; + + if (obj == Py_None || obj == Py_True || obj == Py_False) { + PyObject *cstr = _encoded_const(obj); + if (cstr == NULL) + return -1; + return _steal_list_append(rval, cstr); + } + else if (PyString_Check(obj) || PyUnicode_Check(obj)) + { + PyObject *encoded = encoder_encode_string(s, obj); + if (encoded == NULL) + return -1; + return _steal_list_append(rval, encoded); + } + else if (PyInt_Check(obj) || PyLong_Check(obj)) { + PyObject *encoded = PyObject_Str(obj); + if (encoded == NULL) + return -1; + return _steal_list_append(rval, encoded); + } + else if (PyFloat_Check(obj)) { + PyObject *encoded = encoder_encode_float(s, obj); + if (encoded == NULL) + return -1; + return _steal_list_append(rval, encoded); + } + else if (PyList_Check(obj) || PyTuple_Check(obj)) { + return encoder_listencode_list(s, rval, obj, indent_level); + } + else if (PyDict_Check(obj)) { + return encoder_listencode_dict(s, rval, obj, indent_level); + } + else { + PyObject *ident = NULL; + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(obj); + if (ident == NULL) + return -1; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + Py_DECREF(ident); + return -1; + } + if (PyDict_SetItem(s->markers, ident, obj)) { + Py_DECREF(ident); + return -1; + } + } + newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); + if (newobj == NULL) { + Py_XDECREF(ident); + return -1; + } + rv = encoder_listencode_obj(s, rval, newobj, indent_level); + Py_DECREF(newobj); + if (rv) { + Py_XDECREF(ident); + return -1; + } + if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) { + Py_XDECREF(ident); + return -1; + } + Py_XDECREF(ident); + } + return rv; + } +} + +static int +encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) +{ + /* Encode Python dict dct a JSON term, rval is a PyList */ + static PyObject *open_dict = NULL; + static PyObject *close_dict = NULL; + static PyObject *empty_dict = NULL; + PyObject *kstr = NULL; + PyObject *ident = NULL; + PyObject *key, *value; + Py_ssize_t pos; + int skipkeys; + Py_ssize_t idx; + + if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { + open_dict = PyString_InternFromString("{"); + close_dict = PyString_InternFromString("}"); + empty_dict = PyString_InternFromString("{}"); + if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) + return -1; + } + if (PyDict_Size(dct) == 0) + return PyList_Append(rval, empty_dict); + + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(dct); + if (ident == NULL) + goto bail; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + goto bail; + } + if (PyDict_SetItem(s->markers, ident, dct)) { + goto bail; + } + } + + if (PyList_Append(rval, open_dict)) + goto bail; + + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level += 1; + /* + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + separator = _item_separator + newline_indent + buf += newline_indent + */ + } + + /* TODO: C speedup not implemented for sort_keys */ + + pos = 0; + skipkeys = PyObject_IsTrue(s->skipkeys); + idx = 0; + while (PyDict_Next(dct, &pos, &key, &value)) { + PyObject *encoded; + + if (PyString_Check(key) || PyUnicode_Check(key)) { + Py_INCREF(key); + kstr = key; + } + else if (PyFloat_Check(key)) { + kstr = encoder_encode_float(s, key); + if (kstr == NULL) + goto bail; + } + else if (PyInt_Check(key) || PyLong_Check(key)) { + kstr = PyObject_Str(key); + if (kstr == NULL) + goto bail; + } + else if (key == Py_True || key == Py_False || key == Py_None) { + kstr = _encoded_const(key); + if (kstr == NULL) + goto bail; + } + else if (skipkeys) { + continue; + } + else { + /* TODO: include repr of key */ + PyErr_SetString(PyExc_ValueError, "keys must be a string"); + goto bail; + } + + if (idx) { + if (PyList_Append(rval, s->item_separator)) + goto bail; + } + + encoded = encoder_encode_string(s, kstr); + Py_CLEAR(kstr); + if (encoded == NULL) + goto bail; + if (PyList_Append(rval, encoded)) { + Py_DECREF(encoded); + goto bail; + } + Py_DECREF(encoded); + if (PyList_Append(rval, s->key_separator)) + goto bail; + if (encoder_listencode_obj(s, rval, value, indent_level)) + goto bail; + idx += 1; + } + if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) + goto bail; + Py_CLEAR(ident); + } + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level -= 1; + /* + yield '\n' + (' ' * (_indent * _current_indent_level)) + */ + } + if (PyList_Append(rval, close_dict)) + goto bail; + return 0; + +bail: + Py_XDECREF(kstr); + Py_XDECREF(ident); + return -1; +} + + +static int +encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) +{ + /* Encode Python list seq to a JSON term, rval is a PyList */ + static PyObject *open_array = NULL; + static PyObject *close_array = NULL; + static PyObject *empty_array = NULL; + PyObject *ident = NULL; + PyObject *s_fast = NULL; + Py_ssize_t num_items; + PyObject **seq_items; + Py_ssize_t i; + + if (open_array == NULL || close_array == NULL || empty_array == NULL) { + open_array = PyString_InternFromString("["); + close_array = PyString_InternFromString("]"); + empty_array = PyString_InternFromString("[]"); + if (open_array == NULL || close_array == NULL || empty_array == NULL) + return -1; + } + ident = NULL; + s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); + if (s_fast == NULL) + return -1; + num_items = PySequence_Fast_GET_SIZE(s_fast); + if (num_items == 0) { + Py_DECREF(s_fast); + return PyList_Append(rval, empty_array); + } + + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(seq); + if (ident == NULL) + goto bail; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + goto bail; + } + if (PyDict_SetItem(s->markers, ident, seq)) { + goto bail; + } + } + + seq_items = PySequence_Fast_ITEMS(s_fast); + if (PyList_Append(rval, open_array)) + goto bail; + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level += 1; + /* + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + separator = _item_separator + newline_indent + buf += newline_indent + */ + } + for (i = 0; i < num_items; i++) { + PyObject *obj = seq_items[i]; + if (i) { + if (PyList_Append(rval, s->item_separator)) + goto bail; + } + if (encoder_listencode_obj(s, rval, obj, indent_level)) + goto bail; + } + if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) + goto bail; + Py_CLEAR(ident); + } + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level -= 1; + /* + yield '\n' + (' ' * (_indent * _current_indent_level)) + */ + } + if (PyList_Append(rval, close_array)) + goto bail; + Py_DECREF(s_fast); + return 0; + +bail: + Py_XDECREF(ident); + Py_DECREF(s_fast); + return -1; +} + +static void +encoder_dealloc(PyObject *self) +{ + /* Deallocate Encoder */ + encoder_clear(self); + Py_TYPE(self)->tp_free(self); +} + +static int +encoder_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + Py_VISIT(s->markers); + Py_VISIT(s->defaultfn); + Py_VISIT(s->encoder); + Py_VISIT(s->indent); + Py_VISIT(s->key_separator); + Py_VISIT(s->item_separator); + Py_VISIT(s->sort_keys); + Py_VISIT(s->skipkeys); + return 0; +} + +static int +encoder_clear(PyObject *self) +{ + /* Deallocate Encoder */ + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + Py_CLEAR(s->markers); + Py_CLEAR(s->defaultfn); + Py_CLEAR(s->encoder); + Py_CLEAR(s->indent); + Py_CLEAR(s->key_separator); + Py_CLEAR(s->item_separator); + Py_CLEAR(s->sort_keys); + Py_CLEAR(s->skipkeys); + return 0; +} + +PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); + +static +PyTypeObject PyEncoderType = { + PyObject_HEAD_INIT(NULL) + 0, /* tp_internal */ + "simplejson._speedups.Encoder", /* tp_name */ + sizeof(PyEncoderObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + encoder_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + encoder_call, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + encoder_doc, /* tp_doc */ + encoder_traverse, /* tp_traverse */ + encoder_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + encoder_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + encoder_init, /* tp_init */ + 0, /* tp_alloc */ + encoder_new, /* tp_new */ + 0, /* tp_free */ +}; + static PyMethodDef speedups_methods[] = { - DEFN(encode_basestring_ascii, METH_O), - {} + {"encode_basestring_ascii", + (PyCFunction)py_encode_basestring_ascii, + METH_O, + pydoc_encode_basestring_ascii}, + {"scanstring", + (PyCFunction)py_scanstring, + METH_VARARGS, + pydoc_scanstring}, + {NULL, NULL, 0, NULL} }; -#undef DEFN +PyDoc_STRVAR(module_doc, +"simplejson speedups\n"); + void init_speedups(void) { PyObject *m; - m = Py_InitModule4("_speedups", speedups_methods, NULL, NULL, PYTHON_API_VERSION); + PyScannerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyScannerType) < 0) + return; + PyEncoderType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyEncoderType) < 0) + return; + m = Py_InitModule3("_speedups", speedups_methods, module_doc); + Py_INCREF((PyObject*)&PyScannerType); + PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); + Py_INCREF((PyObject*)&PyEncoderType); + PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); } Modified: trunk/pywikipedia/simplejson/decoder.py =================================================================== --- trunk/pywikipedia/simplejson/decoder.py 2009-08-07 17:25:10 UTC (rev 7129) +++ trunk/pywikipedia/simplejson/decoder.py 2009-08-08 00:42:34 UTC (rev 7130) @@ -1,15 +1,20 @@ +"""Implementation of JSONDecoder """ -Implementation of JSONDecoder -""" import re +import sys +import struct -from simplejson.scanner import Scanner, pattern +from simplejson.scanner import make_scanner +try: + from simplejson._speedups import scanstring as c_scanstring +except ImportError: + c_scanstring = None +__all__ = ['JSONDecoder'] + FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL def _floatconstants(): - import struct - import sys _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') if sys.byteorder != 'big': _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] @@ -18,6 +23,7 @@ NaN, PosInf, NegInf = _floatconstants() + def linecol(doc, pos): lineno = doc.count('\n', 0, pos) + 1 if lineno == 1: @@ -26,38 +32,29 @@ colno = pos - doc.rindex('\n', 0, pos) return lineno, colno + def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups lineno, colno = linecol(doc, pos) if end is None: - return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos) + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) endlineno, endcolno = linecol(doc, end) - return '%s: line %d column %d - line %d column %d (char %d - %d)' % ( - msg, lineno, colno, endlineno, endcolno, pos, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + _CONSTANTS = { '-Infinity': NegInf, 'Infinity': PosInf, 'NaN': NaN, - 'true': True, - 'false': False, - 'null': None, } -def JSONConstant(match, context, c=_CONSTANTS): - return c[match.group(0)], None -pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant) - -def JSONNumber(match, context): - match = JSONNumber.regex.match(match.string, *match.span()) - integer, frac, exp = match.groups() - if frac or exp: - res = float(integer + (frac or '') + (exp or '')) - else: - res = int(integer) - return res, None -pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber) - -STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS) +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': u'"', '\\': u'\\', '/': u'/', 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', @@ -65,7 +62,15 @@ DEFAULT_ENCODING = "utf-8" -def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match): +def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" if encoding is None: encoding = DEFAULT_ENCODING chunks = [] @@ -78,127 +83,184 @@ errmsg("Unterminated string starting at", s, begin)) end = chunk.end() content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters if content: if not isinstance(content, unicode): content = unicode(content, encoding) _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows if terminator == '"': break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise ValueError(errmsg(msg, s, end)) + else: + _append(terminator) + continue try: esc = s[end] except IndexError: raise ValueError( errmsg("Unterminated string starting at", s, begin)) + # If not a unicode escape sequence, must be in the lookup table if esc != 'u': try: - m = _b[esc] + char = _b[esc] except KeyError: - raise ValueError( - errmsg("Invalid \\escape: %r" % (esc,), s, end)) + msg = "Invalid \\escape: " + repr(esc) + raise ValueError(errmsg(msg, s, end)) end += 1 else: + # Unicode escape sequence esc = s[end + 1:end + 5] - try: - m = unichr(int(esc, 16)) - if len(esc) != 4 or not esc.isalnum(): - raise ValueError - except ValueError: - raise ValueError(errmsg("Invalid \\uXXXX escape", s, end)) - end += 5 - _append(m) + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise ValueError(errmsg(msg, s, end)) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise ValueError(errmsg(msg, s, end)) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise ValueError(errmsg(msg, s, end)) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) return u''.join(chunks), end -def JSONString(match, context): - encoding = getattr(context, 'encoding', None) - return scanstring(match.string, match.end(), encoding) -pattern(r'"')(JSONString) -WHITESPACE = re.compile(r'\s*', FLAGS) +# Use speedup if available +scanstring = c_scanstring or py_scanstring -def JSONObject(match, context, _w=WHITESPACE.match): +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): pairs = {} - s = match.string - end = _w(s, match.end()).end() + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty nextchar = s[end:end + 1] - # trivial empty object - if nextchar == '}': - return pairs, end + 1 + # Normally we expect nextchar == '"' if nextchar != '"': - raise ValueError(errmsg("Expecting property name", s, end)) + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + return pairs, end + 1 + elif nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end)) end += 1 - encoding = getattr(context, 'encoding', None) - iterscan = JSONScanner.iterscan while True: - key, end = scanstring(s, end, encoding) - end = _w(s, end).end() + key, end = scanstring(s, end, encoding, strict) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". if s[end:end + 1] != ':': - raise ValueError(errmsg("Expecting : delimiter", s, end)) - end = _w(s, end + 1).end() + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise ValueError(errmsg("Expecting : delimiter", s, end)) + + end += 1 + try: - value, end = iterscan(s, idx=end, context=context).next() + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) except StopIteration: raise ValueError(errmsg("Expecting object", s, end)) pairs[key] = value - end = _w(s, end).end() - nextchar = s[end:end + 1] + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' end += 1 + if nextchar == '}': break - if nextchar != ',': + elif nextchar != ',': raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) - end = _w(s, end).end() - nextchar = s[end:end + 1] + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 if nextchar != '"': raise ValueError(errmsg("Expecting property name", s, end - 1)) - object_hook = getattr(context, 'object_hook', None) + if object_hook is not None: pairs = object_hook(pairs) return pairs, end -pattern(r'{')(JSONObject) - -def JSONArray(match, context, _w=WHITESPACE.match): + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): values = [] - s = match.string - end = _w(s, match.end()).end() - # look-ahead for trivial empty array nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array if nextchar == ']': return values, end + 1 - iterscan = JSONScanner.iterscan + _append = values.append while True: try: - value, end = iterscan(s, idx=end, context=context).next() + value, end = scan_once(s, end) except StopIteration: raise ValueError(errmsg("Expecting object", s, end)) - values.append(value) - end = _w(s, end).end() + _append(value) nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] end += 1 if nextchar == ']': break - if nextchar != ',': + elif nextchar != ',': raise ValueError(errmsg("Expecting , delimiter", s, end)) - end = _w(s, end).end() + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + return values, end -pattern(r'\[')(JSONArray) - -ANYTHING = [ - JSONObject, - JSONArray, - JSONString, - JSONConstant, - JSONNumber, -] -JSONScanner = Scanner(ANYTHING) - class JSONDecoder(object): - """ - Simple JSON <http://json.org> decoder + """Simple JSON <http://json.org> decoder - Performs the following translations in decoding: - + Performs the following translations in decoding by default: + +---------------+-------------------+ | JSON | Python | +===============+===================+ @@ -221,17 +283,15 @@ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. + """ - _scanner = Scanner(ANYTHING) - __all__ = ['__init__', 'decode', 'raw_decode'] - - def __init__(self, encoding=None, object_hook=None): - """ - ``encoding`` determines the encoding used to interpret any ``str`` + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True): + """``encoding`` determines the encoding used to interpret any ``str`` objects decoded by this instance (utf-8 by default). It has no effect when decoding ``unicode`` objects. - + Note that currently only encodings that are a superset of ASCII work, strings of other encodings should be passed in as ``unicode``. @@ -239,14 +299,38 @@ of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). + + ``parse_float``, if specified, will be called with the string + of every JSON float to be decoded. By default this is equivalent to + float(num_str). This can be used to use another datatype or parser + for JSON floats (e.g. decimal.Decimal). + + ``parse_int``, if specified, will be called with the string + of every JSON int to be decoded. By default this is equivalent to + int(num_str). This can be used to use another datatype or parser + for JSON integers (e.g. float). + + ``parse_constant``, if specified, will be called with one of the + following strings: -Infinity, Infinity, NaN. + This can be used to raise an exception if invalid JSON numbers + are encountered. + """ self.encoding = encoding self.object_hook = object_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.scan_once = make_scanner(self) def decode(self, s, _w=WHITESPACE.match): - """ - Return the Python representation of ``s`` (a ``str`` or ``unicode`` + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) + """ obj, end = self.raw_decode(s, idx=_w(s, 0).end()) end = _w(s, end).end() @@ -254,20 +338,17 @@ raise ValueError(errmsg("Extra data", s, end, len(s))) return obj - def raw_decode(self, s, **kw): - """ - Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. + """ - kw.setdefault('context', self) try: - obj, end = self._scanner.iterscan(s, **kw).next() + obj, end = self.scan_once(s, idx) except StopIteration: raise ValueError("No JSON object could be decoded") return obj, end - -__all__ = ['JSONDecoder'] Modified: trunk/pywikipedia/simplejson/encoder.py =================================================================== --- trunk/pywikipedia/simplejson/encoder.py 2009-08-07 17:25:10 UTC (rev 7129) +++ trunk/pywikipedia/simplejson/encoder.py 2009-08-08 00:42:34 UTC (rev 7130) @@ -1,17 +1,20 @@ +"""Implementation of JSONEncoder """ -Implementation of JSONEncoder -""" import re + try: - from simplejson import _speedups + from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii except ImportError: - _speedups = None + c_encode_basestring_ascii = None +try: + from simplejson._speedups import make_encoder as c_make_encoder +except ImportError: + c_make_encoder = None -ESCAPE = re.compile(r'[\x00-\x19\\"\b\f\n\r\t]') -ESCAPE_ASCII = re.compile(r'([\\"/]|[^\ -~])') +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') ESCAPE_DCT = { - # escape all forward slashes to prevent </script> attack - '/': '\\/', '\\': '\\\\', '"': '\\"', '\b': '\\b', @@ -21,40 +24,28 @@ '\t': '\\t', } for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -# assume this produces an infinity on all machines (probably not guaranteed) +# Assume this produces an infinity on all machines (probably not guaranteed) INFINITY = float('1e66666') +FLOAT_REPR = repr -def floatstr(o, allow_nan=True): - # Check for specials. Note that this type of test is processor- and/or - # platform-specific, so do tests which don't depend on the internals. +def encode_basestring(s): + """Return a JSON representation of a Python string - if o != o: - text = 'NaN' - elif o == INFINITY: - text = 'Infinity' - elif o == -INFINITY: - text = '-Infinity' - else: - return repr(o) - - if not allow_nan: - raise ValueError("Out of range float values are not JSON compliant: %r" - % (o,)) - - return text - - -def encode_basestring(s): """ - Return a JSON representation of a Python string - """ def replace(match): return ESCAPE_DCT[match.group(0)] return '"' + ESCAPE.sub(replace, s) + '"' -def encode_basestring_ascii(s): + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') def replace(match): s = match.group(0) try: @@ -62,27 +53,25 @@ except KeyError: n = ord(s) if n < 0x10000: + #return '\\u{0:04x}'.format(n) return '\\u%04x' % (n,) else: # surrogate pair n -= 0x10000 s1 = 0xd800 | ((n >> 10) & 0x3ff) s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) return '\\u%04x\\u%04x' % (s1, s2) return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - -try: - encode_basestring_ascii = _speedups.encode_basestring_ascii - _need_utf8 = True -except AttributeError: - _need_utf8 = False + +encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii + class JSONEncoder(object): - """ - Extensible JSON <http://json.org> encoder for Python data structures. + """Extensible JSON <http://json.org> encoder for Python data structures. Supports the following objects and types by default: - + +-------------------+---------------+ | Python | JSON | +===================+===============+ @@ -105,35 +94,34 @@ ``.default()`` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise ``TypeError``). + """ - __all__ = ['__init__', 'default', 'encode', 'iterencode'] item_separator = ', ' key_separator = ': ' def __init__(self, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8'): - """ - Constructor for JSONEncoder, with sensible defaults. + indent=None, separators=None, encoding='utf-8', default=None): + """Constructor for JSONEncoder, with sensible defaults. - If skipkeys is False, then it is a TypeError to attempt + If skipkeys is false, then it is a TypeError to attempt encoding of keys that are not str, int, long, float or None. If skipkeys is True, such items are simply skipped. - If ensure_ascii is True, the output is guaranteed to be str + If ensure_ascii is true, the output is guaranteed to be str objects with all incoming unicode characters escaped. If ensure_ascii is false, the output will be unicode object. - If check_circular is True, then lists, dicts, and custom encoded + If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to prevent an infinite recursion (which would cause an OverflowError). Otherwise, no such check takes place. - If allow_nan is True, then NaN, Infinity, and -Infinity will be + If allow_nan is true, then NaN, Infinity, and -Infinity will be encoded as such. This behavior is not JSON specification compliant, but is consistent with most JavaScript based encoders and decoders. Otherwise, it will be a ValueError to encode such floats. - If sort_keys is True, then the output of dictionaries will be + If sort_keys is true, then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. @@ -143,12 +131,17 @@ None is the most compact representation. If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON + tuple. The default is (', ', ': '). To get the most compact JSON representation you should specify (',', ':') to eliminate whitespace. + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. + transformed into unicode using that encoding prior to JSON-encoding. The default is UTF-8. + """ self.skipkeys = skipkeys @@ -157,15 +150,133 @@ self.allow_nan = allow_nan self.sort_keys = sort_keys self.indent = indent - self.current_indent_level = 0 if separators is not None: self.item_separator, self.key_separator = separators + if default is not None: + self.default = default self.encoding = encoding - def _newline_indent(self): - return '\n' + (' ' * (self.indent * self.current_indent_level)) + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). - def _iterencode_list(self, lst, markers=None): + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + return ''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): + # Check for specials. Note that this type of test is processor- and/or + # platform-specific, so do tests which don't depend on the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys: + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot) + return _iterencode(o, 0) + +def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, + ## HACK: hand-optimized bytecode; turn globals into locals + False=False, + True=True, + ValueError=ValueError, + basestring=basestring, + dict=dict, + float=float, + id=id, + int=int, + isinstance=isinstance, + list=list, + long=long, + str=str, + tuple=tuple, + ): + + def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' return @@ -174,31 +285,51 @@ if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = lst - yield '[' - if self.indent is not None: - self.current_indent_level += 1 - newline_indent = self._newline_indent() - separator = self.item_separator + newline_indent - yield newline_indent + buf = '[' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + separator = _item_separator + newline_indent + buf += newline_indent else: newline_indent = None - separator = self.item_separator + separator = _item_separator first = True for value in lst: if first: first = False else: - yield separator - for chunk in self._iterencode(value, markers): - yield chunk + buf = separator + if isinstance(value, basestring): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, (int, long)): + yield buf + str(value) + elif isinstance(value, float): + yield buf + _floatstr(value) + else: + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + for chunk in chunks: + yield chunk if newline_indent is not None: - self.current_indent_level -= 1 - yield self._newline_indent() + _current_indent_level -= 1 + yield '\n' + (' ' * (_indent * _current_indent_level)) yield ']' if markers is not None: del markers[markerid] - def _iterencode_dict(self, dct, markers=None): + def _iterencode_dict(dct, _current_indent_level): if not dct: yield '{}' return @@ -208,78 +339,76 @@ raise ValueError("Circular reference detected") markers[markerid] = dct yield '{' - key_separator = self.key_separator - if self.indent is not None: - self.current_indent_level += 1 - newline_indent = self._newline_indent() - item_separator = self.item_separator + newline_indent + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + item_separator = _item_separator + newline_indent yield newline_indent else: newline_indent = None - item_separator = self.item_separator + item_separator = _item_separator first = True - if self.ensure_ascii: - encoder = encode_basestring_ascii + if _sort_keys: + items = dct.items() + items.sort(key=lambda kv: kv[0]) else: - encoder = encode_basestring - allow_nan = self.allow_nan - if self.sort_keys: - keys = dct.keys() - keys.sort() - items = [(k, dct[k]) for k in keys] - else: items = dct.iteritems() - _encoding = self.encoding - _do_decode = (_encoding is not None - and not (_need_utf8 and _encoding == 'utf-8')) for key, value in items: - if isinstance(key, str): - if _do_decode: - key = key.decode(_encoding) - elif isinstance(key, basestring): + if isinstance(key, basestring): pass # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. elif isinstance(key, float): - key = floatstr(key, allow_nan) - elif isinstance(key, (int, long)): - key = str(key) + key = _floatstr(key) elif key is True: key = 'true' elif key is False: key = 'false' elif key is None: key = 'null' - elif self.skipkeys: + elif isinstance(key, (int, long)): + key = str(key) + elif _skipkeys: continue else: - raise TypeError("key %r is not a string" % (key,)) + raise TypeError("key " + repr(key) + " is not a string") if first: first = False else: yield item_separator - yield encoder(key) - yield key_separator - for chunk in self._iterencode(value, markers): - yield chunk + yield _encoder(key) + yield _key_separator + if isinstance(value, basestring): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, (int, long)): + yield str(value) + elif isinstance(value, float): + yield _floatstr(value) + else: + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + for chunk in chunks: + yield chunk if newline_indent is not None: - self.current_indent_level -= 1 - yield self._newline_indent() + _current_indent_level -= 1 + yield '\n' + (' ' * (_indent * _current_indent_level)) yield '}' if markers is not None: del markers[markerid] - def _iterencode(self, o, markers=None): + def _iterencode(o, _current_indent_level): if isinstance(o, basestring): - if self.ensure_ascii: - encoder = encode_basestring_ascii - else: - encoder = encode_basestring - _encoding = self.encoding - if (_encoding is not None and isinstance(o, str) - and not (_need_utf8 and _encoding == 'utf-8')): - o = o.decode(_encoding) - yield encoder(o) + yield _encoder(o) elif o is None: yield 'null' elif o is True: @@ -289,12 +418,12 @@ elif isinstance(o, (int, long)): yield str(o) elif isinstance(o, float): - yield floatstr(o, self.allow_nan) + yield _floatstr(o) elif isinstance(o, (list, tuple)): - for chunk in self._iterencode_list(o, markers): + for chunk in _iterencode_list(o, _current_indent_level): yield chunk elif isinstance(o, dict): - for chunk in self._iterencode_dict(o, markers): + for chunk in _iterencode_dict(o, _current_indent_level): yield chunk else: if markers is not None: @@ -302,70 +431,10 @@ if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - for chunk in self._iterencode_default(o, markers): + o = _default(o) + for chunk in _iterencode(o, _current_indent_level): yield chunk if markers is not None: del markers[markerid] - def _iterencode_default(self, o, markers=None): - newobj = self.default(o) - return self._iterencode(newobj, markers) - - def default(self, o): - """ - Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - """ - raise TypeError("%r is not JSON serializable" % (o,)) - - def encode(self, o): - """ - Return a JSON string representation of a Python data structure. - - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo":["bar", "baz"]}' - """ - # This is for extremely simple cases and benchmarks... - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8' and _need_utf8)): - o = o.decode(_encoding) - return encode_basestring_ascii(o) - # This doesn't pass the iterator directly to ''.join() because it - # sucks at reporting exceptions. It's going to do this internally - # anyway because it uses PySequence_Fast or similar. - chunks = list(self.iterencode(o)) - return ''.join(chunks) - - def iterencode(self, o): - """ - Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - """ - if self.check_circular: - markers = {} - else: - markers = None - return self._iterencode(o, markers) - -__all__ = ['JSONEncoder'] + return _iterencode Modified: trunk/pywikipedia/simplejson/scanner.py =================================================================== --- trunk/pywikipedia/simplejson/scanner.py 2009-08-07 17:25:10 UTC (rev 7129) +++ trunk/pywikipedia/simplejson/scanner.py 2009-08-08 00:42:34 UTC (rev 7130) @@ -1,63 +1,65 @@ +"""JSON token scanner """ -Iterator based sre token scanner -""" -import sre_parse, sre_compile, sre_constants -from sre_constants import BRANCH, SUBPATTERN -from re import VERBOSE, MULTILINE, DOTALL import re +try: + from simplejson._speedups import make_scanner as c_make_scanner +except ImportError: + c_make_scanner = None -__all__ = ['Scanner', 'pattern'] +__all__ = ['make_scanner'] -FLAGS = (VERBOSE | MULTILINE | DOTALL) -class Scanner(object): - def __init__(self, lexicon, flags=FLAGS): - self.actions = [None] - # combine phrases into a compound pattern - s = sre_parse.Pattern() - s.flags = flags - p = [] - for idx, token in enumerate(lexicon): - phrase = token.pattern - try: - subpattern = sre_parse.SubPattern(s, - [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) - except sre_constants.error: - raise - p.append(subpattern) - self.actions.append(token) +NUMBER_RE = re.compile( + r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + (re.VERBOSE | re.MULTILINE | re.DOTALL)) - p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) - self.scanner = sre_compile.compile(p) +def py_make_scanner(context): + parse_object = context.parse_object + parse_array = context.parse_array + parse_string = context.parse_string + match_number = NUMBER_RE.match + encoding = context.encoding + strict = context.strict + parse_float = context.parse_float + parse_int = context.parse_int + parse_constant = context.parse_constant + object_hook = context.object_hook + def _scan_once(string, idx): + try: + nextchar = string[idx] + except IndexError: + raise StopIteration - def iterscan(self, string, idx=0, context=None): - """ - Yield match, end_idx for each match - """ - match = self.scanner.scanner(string, idx).match - actions = self.actions - lastend = idx - end = len(string) - while True: - m = match() - if m is None: - break - matchbegin, matchend = m.span() - if lastend == matchend: - break - action = actions[m.lastindex] - if action is not None: - rval, next_pos = action(m, context) - if next_pos is not None and next_pos != matchend: - # "fast forward" the scanner - matchend = next_pos - match = self.scanner.scanner(string, matchend).match - yield rval, matchend - lastend = matchend - -def pattern(pattern, flags=FLAGS): - def decorator(fn): - fn.pattern = pattern - fn.regex = re.compile(pattern, flags) - return fn - return decorator + if nextchar == '"': + return parse_string(string, idx + 1, encoding, strict) + elif nextchar == '{': + return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook) + elif nextchar == '[': + return parse_array((string, idx + 1), _scan_once) + elif nextchar == 'n' and string[idx:idx + 4] == 'null': + return None, idx + 4 + elif nextchar == 't' and string[idx:idx + 4] == 'true': + return True, idx + 4 + elif nextchar == 'f' and string[idx:idx + 5] == 'false': + return False, idx + 5 + + m = match_number(string, idx) + if m is not None: + integer, frac, exp = m.groups() + if frac or exp: + res = parse_float(integer + (frac or '') + (exp or '')) + else: + res = parse_int(integer) + return res, m.end() + elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': + return parse_constant('NaN'), idx + 3 + elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': + return parse_constant('Infinity'), idx + 8 + elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': + return parse_constant('-Infinity'), idx + 9 + else: + raise StopIteration + + return _scan_once + +make_scanner = c_make_scanner or py_make_scanner Added: trunk/pywikipedia/simplejson/tool.py =================================================================== --- trunk/pywikipedia/simplejson/tool.py (rev 0) +++ trunk/pywikipedia/simplejson/tool.py 2009-08-08 00:42:34 UTC (rev 7130) @@ -0,0 +1,37 @@ +r"""Command-line tool to validate and pretty-print JSON + +Usage:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) + +""" +import sys +import simplejson + +def main(): + if len(sys.argv) == 1: + infile = sys.stdin + outfile = sys.stdout + elif len(sys.argv) == 2: + infile = open(sys.argv[1], 'rb') + outfile = sys.stdout + elif len(sys.argv) == 3: + infile = open(sys.argv[1], 'rb') + outfile = open(sys.argv[2], 'wb') + else: + raise SystemExit(sys.argv[0] + " [infile [outfile]]") + try: + obj = simplejson.load(infile) + except ValueError, e: + raise SystemExit(e) + simplejson.dump(obj, outfile, sort_keys=True, indent=4) + outfile.write('\n') + + +if __name__ == '__main__': + main()

1 0

SVN: [7129] trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py
by siebrand＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7129 Author: siebrand Date: 2009-08-07 17:25:10 +0000 (Fri, 07 Aug 2009) Log Message: ----------- Update documentation - missed details Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py Modified: trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py 2009-08-07 17:15:59 UTC (rev 7128) +++ trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py 2009-08-07 17:25:10 UTC (rev 7129) @@ -3,13 +3,14 @@ import re class DummyReplacement(object): + # only overrides delinks inside <gallery> tags. hook = 'gallery_replace' def __init__(self, CommonsDelinker): self.CommonsDelinker = CommonsDelinker def __call__(self, page, summary, image, replacement, match, groups): site = page.site() if (site.lang, site.family.name) == ('nl', 'wikipedia') and replacement.get() is None: - # Do not delink from pages in categories mentioned in [[User:CommonsDelinker/DummyReplacements]] + # Do not delink inside galleries of pages in categories mentioned in [[User:CommonsDelinker/DummyReplacements]] # but replace the target file with another file (used in for example flag or coat of arms galleries). # Format of the configuration page (see example on http://nl.wikipedia.org/wiki/Gebruiker:CommonsDelinker/DummyReplacements): #

1 0

SVN: [7128] trunk/pywikipedia/commonsdelinker/plugins
by siebrand＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7128 Author: siebrand Date: 2009-08-07 17:15:59 +0000 (Fri, 07 Aug 2009) Log Message: ----------- * rename flags.py to more generic replace_with_dummy.py. Requires reconfiguration if you are using this (f.e. CommonsDelinker['plugins'] = ['replace_with_dummy.DummyReplacement'] * add de.wikipedia to replace_with_dummy.py ([[User:CommonsDelinker/DummyReplacements]]) * rename configuration page for nl.wp to [[User:CommonsDelinker/DummyReplacements]] * change tags from "[begin|end]-flags" to "[begin|end]-flags" * added configuration documentation in replace_with_dummy.py * add svn keyword Id * remove trailing whitespace Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/plugins/__init__.py trunk/pywikipedia/commonsdelinker/plugins/books.py trunk/pywikipedia/commonsdelinker/plugins/debug.py trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py Added Paths: ----------- trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py Removed Paths: ------------- trunk/pywikipedia/commonsdelinker/plugins/flags.py Property Changed: ---------------- trunk/pywikipedia/commonsdelinker/plugins/__init__.py trunk/pywikipedia/commonsdelinker/plugins/books.py trunk/pywikipedia/commonsdelinker/plugins/debug.py trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py Modified: trunk/pywikipedia/commonsdelinker/plugins/__init__.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/__init__.py 2009-08-07 13:33:41 UTC (rev 7127) +++ trunk/pywikipedia/commonsdelinker/plugins/__init__.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -1 +1 @@ -__version__ = '$Id: $' +__version__ = '$Id$' Property changes on: trunk/pywikipedia/commonsdelinker/plugins/__init__.py ___________________________________________________________________ Added: svn:keywords + Id Modified: trunk/pywikipedia/commonsdelinker/plugins/books.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/books.py 2009-08-07 13:33:41 UTC (rev 7127) +++ trunk/pywikipedia/commonsdelinker/plugins/books.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -1,4 +1,4 @@ -__version__ = '$Id: $' +__version__ = '$Id$' import re Property changes on: trunk/pywikipedia/commonsdelinker/plugins/books.py ___________________________________________________________________ Added: svn:keywords + Id Modified: trunk/pywikipedia/commonsdelinker/plugins/debug.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/debug.py 2009-08-07 13:33:41 UTC (rev 7127) +++ trunk/pywikipedia/commonsdelinker/plugins/debug.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -1,5 +1,5 @@ import difflib -__version__ = '$Id: $' +__version__ = '$Id$' class Diff(object): hook = 'before_save' @@ -9,9 +9,9 @@ diff = difflib.context_diff( text.encode('utf-8').splitlines(True), new_text.get().encode('utf-8').splitlines(True)) - + f = open((u'diff/%s-%s-%s.txt' % (page.urlname().replace('/', '-'), page.site().dbName(), page.editTime())).encode('utf-8', 'ignore'), 'w') - + f.writelines(diff) - f.close() \ No newline at end of file + f.close() Property changes on: trunk/pywikipedia/commonsdelinker/plugins/debug.py ___________________________________________________________________ Added: svn:keywords + Id Deleted: trunk/pywikipedia/commonsdelinker/plugins/flags.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/flags.py 2009-08-07 13:33:41 UTC (rev 7127) +++ trunk/pywikipedia/commonsdelinker/plugins/flags.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -1,32 +0,0 @@ -__version__ = '$Id: $' - -import re - -class NlWiki(object): - hook = 'gallery_replace' - def __init__(self, CommonsDelinker): - self.CommonsDelinker = CommonsDelinker - def __call__(self, page, summary, image, replacement, match, groups): - site = page.site() - if (site.lang, site.family.name) == ('nl', 'wikipedia') and replacement.get() is None: - commands = self.CommonsDelinker.SummaryCache.get(site, 'Vlaggen', default = '') - - flags = re.findall(r'(?s)\<\!\-\-begin\-flags (.*?)\-\-\>(.*?)\<\!\-\-end\-flags\-\-\>', commands) - text = page.get() - - namespace = site.namespace(14) - r_namespace = r'(?:[Cc]ategory)|(?:[%s%s]%s)' % \ - (namespace[0], namespace[0].lower(), namespace[1:]) - - for new_image, categories in flags: - for category in categories.split('\n'): - if category.strip() == '': continue - - r_cat = r'\[\[\s*%s\s*\:\s*%s\s*(?:\|.*?)?\s*\]\]' % (r_namespace, - re.sub(r'\\[ _]', '[ _]', re.escape(category.strip()))) - if re.search(r_cat, text): - self.CommonsDelinker.output( - u'%s %s replaced by %s in category %s' % \ - (self, image, new_image, category)) - replacement.set(new_image.replace(' ', '_')) - \ No newline at end of file Copied: trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py (from rev 7123, trunk/pywikipedia/commonsdelinker/plugins/flags.py) =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py (rev 0) +++ trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -0,0 +1,62 @@ +__version__ = '$Id$' + +import re + +class DummyReplacement(object): + hook = 'gallery_replace' + def __init__(self, CommonsDelinker): + self.CommonsDelinker = CommonsDelinker + def __call__(self, page, summary, image, replacement, match, groups): + site = page.site() + if (site.lang, site.family.name) == ('nl', 'wikipedia') and replacement.get() is None: + # Do not delink from pages in categories mentioned in [[User:CommonsDelinker/DummyReplacements]] + # but replace the target file with another file (used in for example flag or coat of arms galleries). + # Format of the configuration page (see example on http://nl.wikipedia.org/wiki/Gebruiker:CommonsDelinker/DummyReplacements): + #  + # Amerikaans zegel + #  + # The above will not delink files in pages in the category "Amerikaans zegel" but replace + # them with the file "Sin escudo.svg" + + commands = self.CommonsDelinker.SummaryCache.get(site, 'DummyReplacements', default = '') + + dummyreplacements = re.findall(r'(?s)\<\!\-\-begin\-dummyreplace (.*?)\-\-\>(.*?)\<\!\-\-end\-dummyreplace\-\-\>', commands) + text = page.get() + + namespace = site.namespace(14) + r_namespace = r'(?:[Cc]ategory)|(?:[%s%s]%s)' % \ + (namespace[0], namespace[0].lower(), namespace[1:]) + + for new_image, categories in dummyreplacements: + for category in categories.split('\n'): + if category.strip() == '': continue + + r_cat = r'\[\[\s*%s\s*\:\s*%s\s*(?:\|.*?)?\s*\]\]' % (r_namespace, + re.sub(r'\\[ _]', '[ _]', re.escape(category.strip()))) + if re.search(r_cat, text): + self.CommonsDelinker.output( + u'%s %s replaced by %s in category %s' % \ + (self, image, new_image, category)) + replacement.set(new_image.replace(' ', '_')) + + if (site.lang, site.family.name) == ('de', 'wikipedia') and replacement.get() is None: + commands = self.CommonsDelinker.SummaryCache.get(site, 'DummyReplacements', default = '') + + dummyreplacements = re.findall(r'(?s)\<\!\-\-begin\-dummyreplace (.*?)\-\-\>(.*?)\<\!\-\-end\-dummyreplace\-\-\>', commands) + text = page.get() + + namespace = site.namespace(14) + r_namespace = r'(?:[Cc]ategory)|(?:[%s%s]%s)' % \ + (namespace[0], namespace[0].lower(), namespace[1:]) + + for new_image, categories in dummyreplacements: + for category in categories.split('\n'): + if category.strip() == '': continue + + r_cat = r'\[\[\s*%s\s*\:\s*%s\s*(?:\|.*?)?\s*\]\]' % (r_namespace, + re.sub(r'\\[ _]', '[ _]', re.escape(category.strip()))) + if re.search(r_cat, text): + self.CommonsDelinker.output( + u'%s %s replaced by %s in category %s' % \ + (self, image, new_image, category)) + replacement.set(new_image.replace(' ', '_')) Property changes on: trunk/pywikipedia/commonsdelinker/plugins/replace_with_dummy.py ___________________________________________________________________ Added: svn:keywords + Id Added: svn:eol-style + native Modified: trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py =================================================================== --- trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py 2009-08-07 13:33:41 UTC (rev 7127) +++ trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py 2009-08-07 17:15:59 UTC (rev 7128) @@ -1,4 +1,4 @@ -__version__ = '$Id: $' +__version__ = '$Id$' class SkipPages(object): hook = 'before_save' @@ -9,4 +9,4 @@ if (site.lang, site.family.name) == ('en', 'wikipedia'): if page.title == 'Wikipedia:Sound/list': return False - return True \ No newline at end of file + return True Property changes on: trunk/pywikipedia/commonsdelinker/plugins/skipped_pages.py ___________________________________________________________________ Added: svn:keywords + Id

1 0

SVN: [7127] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7127 Author: alexsh Date: 2009-08-07 13:33:41 +0000 (Fri, 07 Aug 2009) Log Message: ----------- * Site().solveCaptcha(): Fix dict key wrong for error code. * Site()._getUserData() API mode:Fix get anonymous key normally, base on ordinary mode. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-07 09:34:48 UTC (rev 7126) +++ trunk/pywikipedia/wikipedia.py 2009-08-07 13:33:41 UTC (rev 7127) @@ -4490,6 +4490,7 @@ """Log the user in if not already logged in.""" if not self.loggedInAs(sysop = sysop): loginMan = login.LoginManager(site = self, sysop = sysop) + #loginMan.logout() if loginMan.login(retry = True): index = self._userIndex(sysop) self._isLoggedIn[index] = True @@ -4623,9 +4624,9 @@ def solveCaptcha(self, data): if type(data) == dict: # API Mode result - if data['result'].has_key("captcha"): - captype = data['result']['captcha']['type'] - id = data['result']['captcha']['id'] + if data.has_key("captcha"): + captype = data['captcha']['type'] + id = data['captcha']['id'] if captype in ['simple', 'math', 'question']: answer = input('What is the answer to the captcha "%s" ?' % data['result']['captcha']['question']) elif captype == 'image': @@ -4969,15 +4970,14 @@ self._rights[index] = list(set(self._rights[index])) # Get token - if text.has_key('preferencestoken') and len(text['preferencestoken']) > 2: - # anonymous token is '+\\', check len('+\\') = 2 - # if preferencestoken > 2, it must be loggedin. + if text.has_key('preferencestoken'): self._token[index] = text['preferencestoken'] if self._rights[index] is not None: # Token and rights are loaded - user data is now loaded self._userData[index] = True else: - output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) + if not self._isBlocked[index]: + output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) else: #ordinary mode to get data from edit page HTMLs and JavaScripts @@ -5209,6 +5209,7 @@ def has_mediawiki_message(self, key): """Return True iff this site defines a MediaWiki message for 'key'.""" + #return self._mediawiki_messages.has_key(key) try: v = self.mediawiki_message(key) return True

1 0

SVN: [7126] trunk/pywikipedia/login.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7126 Author: alexsh Date: 2009-08-07 09:34:48 +0000 (Fri, 07 Aug 2009) Log Message: ----------- Fix: correct ordinary logout SpecialPage Modified Paths: -------------- trunk/pywikipedia/login.py Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2009-08-07 07:59:10 UTC (rev 7125) +++ trunk/pywikipedia/login.py 2009-08-07 09:34:48 UTC (rev 7126) @@ -279,15 +279,13 @@ if query.GetData({'action':'logout'}, self.site) == []: flushCk = True else: - logout_address = self.site.get_address("Special:Version") - text = self.site.getUrl(logout_address) - if wikipedia.mediawiki_message('logouttext') in text: #confirm loggedout + text = self.site.getUrl(self.site.get_address("Special:UserLogout")) + if self.site.mediawiki_message('logouttext') in text: #confirm loggedout flushCk = True if flushCk: filename = wikipedia.config.datafilepath('login-data', - '%s-%s-%s-login.data' - % (self.site.family.name, self.site.lang, self.username)) + '%s-%s-%s-login.data' % (self.site.family.name, self.site.lang, self.username)) try: os.remove(filename) except:

1 0

SVN: [7125] trunk/pywikipedia/login.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7125 Author: alexsh Date: 2009-08-07 07:59:10 +0000 (Fri, 07 Aug 2009) Log Message: ----------- Fix:site NotImplementedError and return again by ordinary. Modified Paths: -------------- trunk/pywikipedia/login.py Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2009-08-07 07:51:37 UTC (rev 7124) +++ trunk/pywikipedia/login.py 2009-08-07 07:59:10 UTC (rev 7125) @@ -251,9 +251,9 @@ try: cookiedata = self.getCookie(api) except NotImplementedError: - wikipedia.output('API disabled because this site does not support.') + wikipedia.output('API disabled because this site does not support.\nRetrying by ordinary way...') api = False - cookiedata = self.getCookie(api) + return self.login(False, retry) if cookiedata: self.storecookiedata(cookiedata) wikipedia.output(u"Should be logged in now") @@ -269,7 +269,7 @@ if retry: self.password = None - return self.login(api = api, retry = True) + return self.login(api, True) else: return False

1 0

SVN: [7124] trunk/pywikipedia/login.py
by alexsh＠svn.wikimedia.org 07 Aug '09

07 Aug '09

Revision: 7124 Author: alexsh Date: 2009-08-07 07:51:37 +0000 (Fri, 07 Aug 2009) Log Message: ----------- login.py: *LoginManager.logout():sometimes put edit could came back badtoken because two or more duplicate login session used in pybot and other browsers, this function could abandon server login session, and delete local cookie data. *LoginManager.getCookie(): change API to pure query..(it will handle other login error codes) *cleanup no necessary syntaxes. Modified Paths: -------------- trunk/pywikipedia/login.py Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2009-08-06 21:57:43 UTC (rev 7123) +++ trunk/pywikipedia/login.py 2009-08-07 07:51:37 UTC (rev 7124) @@ -50,7 +50,7 @@ # __version__='$Id$' -import re +import re, os, query import urllib2 import wikipedia, config @@ -122,7 +122,7 @@ if api: predata = { 'action': 'login', - 'lgname': self.username.encode(self.site.encoding()), + 'lgname': self.username, 'lgpassword': self.password, 'lgdomain': self.site.family.ldapDomain, } @@ -163,14 +163,17 @@ wikipedia.cj.save(wikipedia.COOKIEFILE) return "Ok" else: - response, data = self.site.postData(address, self.site.urlEncode(predata)) - if self.verbose: - fakepredata = predata - fakepredata['wpPassword'] = fakepredata['lgpassword'] = u'XXXXX' - wikipedia.output(u"self.site.postData(%s, %s)" % (address, self.site.urlEncode(fakepredata))) - fakeresponsemsg = re.sub(r"(session|Token)=..........", r"session=XXXXXXXXXX", response.msg.__str__()) - wikipedia.output(u"%s/%s\n%s" % (response.status, response.reason, fakeresponsemsg)) - wikipedia.output(u"%s" % data) + if api: + response, data = query.GetData(predata, self.site, back_response = True) + else: + response, data = self.site.postData(address, self.site.urlEncode(predata)) + if self.verbose: + fakepredata = predata + fakepredata['wpPassword'] = fakepredata['lgpassword'] = u'XXXXX' + wikipedia.output(u"self.site.postData(%s, %s)" % (address, self.site.urlEncode(fakepredata))) + fakeresponsemsg = re.sub(r"(session|Token)=..........", r"session=XXXXXXXXXX", response.msg.__str__()) + wikipedia.output(u"%s/%s\n%s" % (response.status, response.reason, fakeresponsemsg)) + wikipedia.output(u"%s" % data) Reat=re.compile(': (.*?);') L = [] @@ -246,11 +249,11 @@ wikipedia.output(u"Logging in to %s as %s" % (self.site, self.username)) try: - cookiedata = self.getCookie(api = api) + cookiedata = self.getCookie(api) except NotImplementedError: wikipedia.output('API disabled because this site does not support.') - config.use_api_login = api = False - cookiedata = self.getCookie(api = api) + api = False + cookiedata = self.getCookie(api) if cookiedata: self.storecookiedata(cookiedata) wikipedia.output(u"Should be logged in now") @@ -262,17 +265,40 @@ wikipedia.output(u"Login failed. Wrong password or CAPTCHA answer?") if api: wikipedia.output(u"API login failed, retrying using standard webpage.") - return self.login(api = False, retry = retry) + return self.login(False, retry) if retry: self.password = None return self.login(api = api, retry = True) else: return False + + def logout(self, api = config.use_api): + flushCk = False + if api and self.site.versionnumber() >= 12: + if query.GetData({'action':'logout'}, self.site) == []: + flushCk = True + else: + logout_address = self.site.get_address("Special:Version") + text = self.site.getUrl(logout_address) + if wikipedia.mediawiki_message('logouttext') in text: #confirm loggedout + flushCk = True + + if flushCk: + filename = wikipedia.config.datafilepath('login-data', + '%s-%s-%s-login.data' + % (self.site.family.name, self.site.lang, self.username)) + try: + os.remove(filename) + except: + pass + return True + + return False def showCaptchaWindow(self, url): pass - + def main(): username = password = None sysop = False @@ -304,11 +330,12 @@ namedict = config.sysopnames else: namedict = config.usernames + for familyName in namedict.iterkeys(): for lang in namedict[familyName].iterkeys(): try: - site = wikipedia.getSite( code=lang, fam=familyName ) - if not forceLogin and site.loggedInAs(sysop = sysop) is not None: + site = wikipedia.getSite(lang, familyName) + if not forceLogin and site.loggedInAs(sysop = sysop): wikipedia.output(u'Already logged in on %s' % site) else: loginMan = LoginManager(password, sysop = sysop, site = site, verbose=verbose)

1 0

SVN: [7123] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7123 Author: alexsh Date: 2009-08-06 21:57:43 +0000 (Thu, 06 Aug 2009) Log Message: ----------- *Site().mediawiki_message(): make temporary remedial when API query messages got KeyError (turn off API, back to ordinary) *let all query.GetData() use current available Site() in all functions, not create again in query.py Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-06 21:56:05 UTC (rev 7122) +++ trunk/pywikipedia/wikipedia.py 2009-08-06 21:57:43 UTC (rev 7123) @@ -919,7 +919,7 @@ 'prop' :'info', 'titles' :self.title(), } - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self.site(), encodeTitle = False) pageid = data['query']['pages'].keys()[0] if data['query']['pages'][pageid].keys()[0] == 'lastrevid': return data['query']['pages'][pageid]['lastrevid'] # if ok, @@ -951,7 +951,7 @@ 'tllimit' :tllimit, } - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self.site(), encodeTitle = False) try: pageid = data['query']['pages'].keys()[0] except KeyError: @@ -1325,7 +1325,7 @@ #if titles: # predata['titles'] = query.ListToParam(titles) - text = query.GetData(predata)['query']['pages'] + text = query.GetData(predata, self.site())['query']['pages'] for pageid in text: if text[pageid].has_key('missing'): @@ -2784,7 +2784,7 @@ 'titles' :self.title(), } try: - data = query.GetData(params, encodeTitle = False)['query']['pages'] + data = query.GetData(params, self,site(), encodeTitle = False)['query']['pages'] except KeyError: raise NoPage(u'API Error, nothing found in the APIs') @@ -2848,7 +2848,7 @@ 'titles' :self.title(), 'iiprop' :'url', } - imagedata = query.GetData(params, encodeTitle = False) + imagedata = query.GetData(params, self.site(), encodeTitle = False) try: url=imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] # urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) @@ -2918,7 +2918,7 @@ 'prop' :'imageinfo', 'titles' :self.title(), } - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self.site(), encodeTitle = False) try: # We don't know the page's id, if any other better idea please change it pageid = data['query']['pages'].keys()[0] @@ -2940,7 +2940,7 @@ 'iiprop' :'sha1', } # First of all we need the Hash that identify an image - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self.site(), encodeTitle = False) pageid = data['query']['pages'].keys()[0] try: hash_found = data['query']['pages'][pageid][u'imageinfo'][0][u'sha1'] @@ -5103,6 +5103,7 @@ """Return the MediaWiki message text for key "key" """ # Allmessages is retrieved once for all per created Site object if not self._mediawiki_messages: + api = False if verbose: output( u"Retrieving mediawiki messages from Special:Allmessages") @@ -5110,6 +5111,8 @@ if self.versionnumber() < 12: usePHP = True else: + if config.use_api: + api = True usePHP = False elementtree = True try: @@ -5133,20 +5136,22 @@ retry_idle_time = 1 while True: - if config.use_api and self.versionnumber() >= 12: + if api and self.versionnumber() >= 12: params = { 'action':'query', 'meta':'allmessages', } try: - datas = query.GetData(params)['query']['allmessages'] + datas = query.GetData(params, self)['query']['allmessages'] + self._mediawiki_messages = _dict([(tag['name'].lower(), tag['*']) + for tag in datas]) except KeyError: - raise ServerError("The APIs don't return data, the site may be down") + output('API get messages had some error, retrying by ordinary.') + api = False + continue except NotImplementedError: - config.use_api = False + api = False continue - self._mediawiki_messages = _dict([(tag['name'].lower(), tag['*']) - for tag in datas]) elif usePHP: phppage = self.getUrl(self.get_address("Special:Allmessages") + "&ot=php") @@ -5522,7 +5527,7 @@ if leuser is not None: params['leuser'] = leuser if letitle is not None: params['letitle'] = letitle while True: - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self, encodeTitle = False) try: imagesData = data['query']['logevents'] except KeyError: @@ -5590,7 +5595,7 @@ if rcshow is not None: params['rcshow'] = rcshow if rctype is not None: params['rctype'] = rctype while True: - data = query.GetData(params, encodeTitle = False) + data = query.GetData(params, self, encodeTitle = False) try: rcData = data['query']['recentchanges'] except KeyError: @@ -5726,7 +5731,7 @@ 'rnlimit': '1', #'': '', } - data = query.GetData(params) + data = query.GetData(params, self) return Page(self, data['query']['random'][0]['title']) else: """Yield random page via Special:Random""" @@ -5744,7 +5749,7 @@ 'rnlimit': '1', 'rnredirect': '1', } - data = query.GetData(params) + data = query.GetData(params, self) return Page(self, data['query']['random'][0]['title']) else: """Yield random redirect page via Special:RandomRedirect.""" @@ -5798,7 +5803,7 @@ params['apfrom'] = start if throttle: get_throttle() - data = query.GetData(params) + data = query.GetData(params, self) for p in data['query']['allpages']: yield Page(self, p['title']) @@ -5990,7 +5995,7 @@ 'euquery': url, } while True: - data = query.GetData(params) + data = query.GetData(params, self) if data['query']['exturlusage'] == []: break for pages in data['query']['exturlusage']: @@ -6606,7 +6611,7 @@ 'list' :'allimages', 'aisha1' :hash_found, } - allimages = query.GetData(params, getSite(self.lang, self.family), encodeTitle = False)['query']['allimages'] + allimages = query.GetData(params, self, encodeTitle = False)['query']['allimages'] files = list() for imagedata in allimages: image = imagedata[u'name']

1 0

SVN: [7122] trunk/pywikipedia/commonsdelinker/image_replacer.py
by btongminh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7122 Author: btongminh Date: 2009-08-06 21:56:05 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Fix freaky typo Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/image_replacer.py Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-06 18:10:15 UTC (rev 7121) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-06 21:56:05 UTC (rev 7122) @@ -133,7 +133,7 @@ for remove in remove_from_list: text = text.replace(remove, u'') # Kill the freaky CommonsDupes - text = text.replacer('== Dummy section, heading can be deleted (using [http://tools.wikimedia.de/~magnus/commons_dupes.php CommonsDupes]) ==', '') + text = text.replace('== Dummy section, heading can be deleted (using [http://tools.wikimedia.de/~magnus/commons_dupes.php CommonsDupes]) ==', '') # Kill the freaky whitespace text = text.replace('\r', '') while '\n\n\n' in text:

1 0

SVN: [7121] trunk/pywikipedia/welcome.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7121 Author: alexsh Date: 2009-08-06 18:10:15 +0000 (Thu, 06 Aug 2009) Log Message: ----------- welcome.py.blocked(): make simple to return user is blocked Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-08-06 17:54:49 UTC (rev 7120) +++ trunk/pywikipedia/welcome.py 2009-08-06 18:10:15 UTC (rev 7121) @@ -497,22 +497,16 @@ #action=query&list=users&ususers=Filnik&usprop=blockinfo """ Function that detects if a user is currently blocked or not. - """ - + """ params = { - 'action' :'query', - 'list' :'users', - 'ususers' :username, - 'usprop' :'blockinfo', - } + 'action': 'query', + 'list': 'users', + 'ususers': username, + 'usprop': 'blockinfo', + } - data = query.GetData(params, encodeTitle = False) # If there's not the blockedby parameter (that means the user isn't blocked), it will return False otherwise True. - try: - blockedBy = data['query']['users'][0]['blockedby'] - except KeyError: - return False # No he's not - return True # Yes is blocked + return query.GetData(params, encodeTitle = False)['query']['users'][0].has_key('blockedby') def defineSign(wsite, signPageTitle, fileSignName = None, fileOption = False): """ Function to load the random signatures. """

1 0

SVN: [7120] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7120 Author: alexsh Date: 2009-08-06 17:54:49 +0000 (Thu, 06 Aug 2009) Log Message: ----------- patch bug #2832749 #2820797 Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/featured.py trunk/pywikipedia/lonelypages.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -652,6 +652,7 @@ 'an': [u'Desambig', u'Disambig'], 'ar': [u'Disambig', u'توضيح'], 'arc': [u'ܕ'], + 'arz': [u'توضيح'], 'ast': [u'Dixebra'], 'av': [u'Неоднозначность'], 'az': [u'Dəqiqləşdirmə'], Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/featured.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -175,6 +175,7 @@ 'ko': (CAT, u"알찬 글"), 'ksh': (CAT, u"Exzälenter Aatikkel"), 'la': (CAT, u"Paginae mensis"), + 'li': (CAT, u"Wikipedia:Sjterartikele"), 'lmo': (CAT, u"Articol ben faa"), 'lo': (CAT, u"ບົດຄວາມດີເດັ່ນ"), 'lt': (CAT, u"Vikipedijos pavyzdiniai straipsniai"), Modified: trunk/pywikipedia/lonelypages.py =================================================================== --- trunk/pywikipedia/lonelypages.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/lonelypages.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -66,6 +66,7 @@ 'ar':u'{{يتيمة|تاريخ={{نسخ:اسم_شهر}} {{نسخ:عام}}}}', 'en':u'{{Orphan|date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}', 'it':u'{{O||mese={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}', + 'ja':u'{{孤立|{{subst:DATE}}}}', 'zh':u'{{subst:Orphan/auto}}', } @@ -74,6 +75,7 @@ 'ar':u'بوت: صفحة يتيمة، إضافة قالب', 'en':u'Bot: Orphan page, add template', 'it':u'Bot: Voce orfana, aggiungo template {{O}}', + 'ja':u'ロボットによる: 孤立したページへのテンプレート貼付け', 'zh':u'機器人: 本頁的鏈入頁面太少', } @@ -83,6 +85,7 @@ 'ar':u'بوت: إضافة صفحة توضيح', 'en':u'Bot: Adding a disambig page', 'it':u'Bot: Aggiungo una disambigua', + 'ja':u'ロボットによる: 曖昧さ回避の追加', 'zh':u'機器人: 增加消歧義頁面', } @@ -93,6 +96,7 @@ 'ar': [ur'\{\{(?:قالب:|)(يتيمة)[\|\}]'], 'en': [r'\{\{(?:template:|)(orphan)[\|\}]', r'\{\{(?:template:|)(wi)[\|\}]'], 'it': [r'\{\{(?:template:|)(o)[\|\}]'], + 'ja': [ur'\{\{(?:template:|)(孤立)[\|\}]'], 'zh': [r'\{\{(?:template:|)(orphan)[\|\}]'], }

1 0

SVN: [7119] trunk/pywikipedia/casechecker.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7119 Author: alexsh Date: 2009-08-06 13:55:25 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Fix:query.getData() parameter Modified Paths: -------------- trunk/pywikipedia/casechecker.py Modified: trunk/pywikipedia/casechecker.py =================================================================== --- trunk/pywikipedia/casechecker.py 2009-08-06 11:08:52 UTC (rev 7118) +++ trunk/pywikipedia/casechecker.py 2009-08-06 13:55:25 UTC (rev 7119) @@ -232,7 +232,7 @@ while True: # Get data self.params['gapfrom'] = self.apfrom - data = query.GetData(self.site.lang, self.params, wikipedia.verbose, True) + data = query.GetData(self.params, self.site, wikipedia.verbose, True) try: self.apfrom = data['query-continue']['allpages']['gapfrom'] except:

1 0

SVN: [7118] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7118 Author: alexsh Date: 2009-08-06 11:08:52 +0000 (Thu, 06 Aug 2009) Log Message: ----------- wikipedia.py: Put API version lock. according [[mw:API:Query]] information (some API query is not necessary because they start from 1.9) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-06 10:45:42 UTC (rev 7117) +++ trunk/pywikipedia/wikipedia.py 2009-08-06 11:08:52 UTC (rev 7118) @@ -5133,7 +5133,7 @@ retry_idle_time = 1 while True: - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action':'query', 'meta':'allmessages', @@ -5718,7 +5718,7 @@ break def randompage(self): - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action': 'query', 'list': 'random', @@ -5736,7 +5736,7 @@ return Page(self, m.group('title')) def randomredirectpage(self): - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action': 'query', 'list': 'random', @@ -5980,7 +5980,7 @@ urlsToRetrieve = [siteurl] if not siteurl.startswith('*.'): urlsToRetrieve.append('*.' + siteurl) - if config.use_api: + if config.use_api and self.versionnumber() >= 11: output(u'Querying API exturlusage...') for url in urlsToRetrieve: params = { @@ -6593,6 +6593,9 @@ NOTE 2: it returns the image WITHOUT the image namespace. """ + if self.versionnumber() < 12: + return None + if hash_found is None: # If the hash is none return None and not continue return None # Now get all the images with the same hash

1 0

SVN: [7117] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7117 Author: alexsh Date: 2009-08-06 10:45:42 +0000 (Thu, 06 Aug 2009) Log Message: ----------- *Site()._getUserData(): add description for collect data from API. *Page().delete(): Add API option to delete page.(tested on redirect.py broken, could reduce more time to load delete page.) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-06 07:15:41 UTC (rev 7116) +++ trunk/pywikipedia/wikipedia.py 2009-08-06 10:45:42 UTC (rev 7117) @@ -2396,50 +2396,71 @@ answer = 'y' self.site()._noDeletePrompt = True if answer == 'y': - host = self.site().hostname() - address = self.site().delete_address(self.urlname()) - - reason = reason.encode(self.site().encoding()) + token = self.site().getToken(self, sysop = True) - predata = { - 'wpDeleteReasonList': 'other', - 'wpReason': reason, - 'wpComment': reason, - 'wpConfirm': '1', - 'wpConfirmB': '1' - } - if token: - predata['wpEditToken'] = token - if self.site().hostname() in config.authenticate.keys(): - predata['Content-type'] = 'application/x-www-form-urlencoded' - predata['User-agent'] = useragent - data = self.site().urlEncode(predata) - response = urllib2.urlopen(urllib2.Request(self.site().protocol() + '://' + self.site().hostname() + address, data)) - data = u'' - else: - response, data = self.site().postForm(address, predata, sysop = True) - if data: - self.site().checkBlocks(sysop = True) - if self.site().mediawiki_message('actioncomplete') in data: + + if config.use_api and self.site().versionnumber() >= 12: + params = { + 'action': 'delete', + 'title': self.title(), + 'token': token, + 'reason': reason, + } + datas = query.GetData(params, self.site(), sysop = True) + if datas.has_key('delete'): output(u'Page %s deleted' % self.aslink(forceInterwiki = True)) return True - elif self.site().mediawiki_message('cannotdelete') in data: - output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) - return False else: - output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) - try: - ibegin = data.index('') + 22 - iend = data.index('') - except ValueError: - # if begin/end markers weren't found, show entire HTML file - output(data) + if datas['error']['code'] == 'missingtitle': + output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) + return False else: - # otherwise, remove the irrelevant sections - data = data[ibegin:iend] - output(data) - return False + output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) + output('%s' % datas) + else: + host = self.site().hostname() + address = self.site().delete_address(self.urlname()) + reason = reason.encode(self.site().encoding()) + predata = { + 'wpDeleteReasonList': 'other', + 'wpReason': reason, + 'wpComment': reason, + 'wpConfirm': '1', + 'wpConfirmB': '1' + } + if token: + predata['wpEditToken'] = token + if self.site().hostname() in config.authenticate.keys(): + predata['Content-type'] = 'application/x-www-form-urlencoded' + predata['User-agent'] = useragent + data = self.site().urlEncode(predata) + response = urllib2.urlopen(urllib2.Request(self.site().protocol() + '://' + self.site().hostname() + address, data)) + data = u'' + else: + response, data = self.site().postForm(address, predata, sysop = True) + if data: + self.site().checkBlocks(sysop = True) + if self.site().mediawiki_message('actioncomplete') in data: + output(u'Page %s deleted' % self.aslink(forceInterwiki = True)) + return True + elif self.site().mediawiki_message('cannotdelete') in data: + output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) + return False + else: + output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) + try: + ibegin = data.index('') + 22 + iend = data.index('') + except ValueError: + # if begin/end markers weren't found, show entire HTML file + output(data) + else: + # otherwise, remove the irrelevant sections + data = data[ibegin:iend] + output(data) + return False + def loadDeletedRevisions(self): """Retrieve all deleted revisions for this Page from Special/Undelete. @@ -4879,8 +4900,7 @@ index = self._userIndex(sysop) if type(text) == dict: #text is dict, query from API - # Check for blocks - but only if version is 1.11 (userinfo is available) - # and the user data was not yet loaded + # Check for blocks if text.has_key('blockedby') and not self._isBlocked[index]: # Write a warning if not shown earlier if sysop: @@ -4890,7 +4910,7 @@ output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) self._isBlocked[index] = text.has_key('blockedby') - # Check for new messages, show key 'messages' in dict. + # Check for new messages, the data must had key 'messages' in dict. if text.has_key('messages'): if not self._messages[index]: # User has *new* messages @@ -4907,16 +4927,16 @@ return # Get username. - # anonymous mode will show key 'anon' + # The data in anonymous mode had key 'anon' + # if 'anon' exist, username is IP address, not to collect it right now if not text.has_key('anon'): self._isLoggedIn[index] = True self._userName[index] = text['name'] else: self._isLoggedIn[index] = False - # No idea what is the user name, and it isn't important self._userName[index] = None - # Check user groups and rights + # Get user groups and rights if text.has_key('groups') and text['groups'] != []: self._rights[index] = text['groups'] self._rights[index].extend(text['rights']) @@ -4933,7 +4953,7 @@ if sysop and 'sysop' not in self._rights[index]: output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) else: - # key groups is not exists, setup a default rights + # 'groups' is not exists, set default rights self._rights[index] = [] if self._isLoggedIn[index]: # Logged in user @@ -4948,13 +4968,13 @@ #remove Duplicate rights self._rights[index] = list(set(self._rights[index])) - # Search for a token + # Get token if text.has_key('preferencestoken') and len(text['preferencestoken']) > 2: # anonymous token is '+\\', check len('+\\') = 2 # if preferencestoken > 2, it must be loggedin. self._token[index] = text['preferencestoken'] if self._rights[index] is not None: - # In this case, token and rights are loaded - user data is now loaded + # Token and rights are loaded - user data is now loaded self._userData[index] = True else: output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) @@ -5213,6 +5233,7 @@ config.use_api = False # Get data + # API Userinfo is available from version 1.11 if config.use_api and self.versionnumber() >= 11: #Query userinfo params = {

1 0

SVN: [7116] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7116 Author: alexsh Date: 2009-08-06 07:15:41 +0000 (Thu, 06 Aug 2009) Log Message: ----------- query.GetData(): Fix postData cookie cannot use sysop data Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-06 06:00:28 UTC (rev 7115) +++ trunk/pywikipedia/query.py 2009-08-06 07:15:41 UTC (rev 7116) @@ -81,7 +81,7 @@ try: jsontext = "Nothing received" if params['action'] in postAC: - res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(), sysop=sysop) + res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(sysop=sysop), sysop=sysop) else: if back_response: res, jsontext = site.getUrl( path, retry=True, data=data, sysop=sysop, back_response=True)

1 0

SVN: [7115] trunk/pywikipedia/replace.py
by purodha＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7115 Author: purodha Date: 2009-08-06 06:00:28 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Add -query:(pagecount) parameter to replace.py - same as interwiki.py has. Modified Paths: -------------- trunk/pywikipedia/replace.py Modified: trunk/pywikipedia/replace.py =================================================================== --- trunk/pywikipedia/replace.py 2009-08-05 21:40:30 UTC (rev 7114) +++ trunk/pywikipedia/replace.py 2009-08-06 06:00:28 UTC (rev 7115) @@ -62,6 +62,9 @@ the bot will check every regex without waiting using all the resources. This will slow it down between a regex and another in order not to waste too much CPU. + +-query: The maximum number of pages that the bot will load at once. + Default value is 60. Ignored when reading an XML file. -fix:XYZ Perform one of the predefined replacements tasks, which are given in the dictionary 'fixes' defined inside the file @@ -495,6 +498,8 @@ allowoverlap = False # Do not recurse replacement recursive = False + # This is the maximum number of pages to load per query + maxquerysize = 60 # This factory is responsible for processing command line arguments # that are also used by other scripts and that determine on which pages # to work on. @@ -561,6 +566,8 @@ summary_commandline = True elif arg.startswith('-allowoverlap'): allowoverlap = True + elif arg.startswith('-query:'): + maxquerysize = int(arg[7:]) else: if not genFactory.handleArg(arg): commandline_replacements.append(arg) @@ -696,7 +703,7 @@ preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=20, lookahead=100) else: - preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=60) + preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=maxquerysize) bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall, allowoverlap, recursive, add_cat, sleep, editSummary) bot.run()

1 0

SVN: [7114] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7114 Author: alexsh Date: 2009-08-05 21:40:30 +0000 (Wed, 05 Aug 2009) Log Message: ----------- Remove all 'useAPI=True' tag for function set useAPI default True Modified Paths: -------------- trunk/pywikipedia/casechecker.py trunk/pywikipedia/catlib.py trunk/pywikipedia/imageuncat.py trunk/pywikipedia/maintenance/family_check.py trunk/pywikipedia/welcome.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/casechecker.py =================================================================== --- trunk/pywikipedia/casechecker.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/casechecker.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -200,14 +200,14 @@ wlpage = self.whitelists[self.site.lang] wikipedia.output(u'Loading whitelist from %s' % wlpage) wlparams = { - 'action' : 'query', - 'prop' : 'links', - 'titles' : wlpage, - 'redirects' : '', - 'indexpageids' : '', - } + 'action' : 'query', + 'prop' : 'links', + 'titles' : wlpage, + 'redirects' : '', + 'indexpageids' : '', + } - data = query.GetData(self.site.lang, wlparams, wikipedia.verbose, useAPI=True, encodeTitle=False) + data = query.GetData(wlparams, wikipedia.getSite(self.site.lang), wikipedia.verbose, encodeTitle=False) if len(data['query']['pageids']) == 1: pageid = data['query']['pageids'][0] links = data['query']['pages'][pageid]['links'] Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/catlib.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -533,8 +533,7 @@ 'cmtitle' :CatName, } - data = query.GetData(params, site = site, - useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) categories_parsed.append(CatName) try: members = data['query']['categorymembers'] Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/imageuncat.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -1243,7 +1243,7 @@ 'leend' :today.strftime(dateformat) } - data = query.GetData(params, site, useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) try: for item in data['query']['logevents']: result.append(item['title']) @@ -1279,7 +1279,7 @@ 'rctype' :'edit|log', } - data = query.GetData(params, site, useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) try: for item in data['query']['recentchanges']: result.append(item['title']) Modified: trunk/pywikipedia/maintenance/family_check.py =================================================================== --- trunk/pywikipedia/maintenance/family_check.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/maintenance/family_check.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -17,7 +17,7 @@ 'meta': 'siteinfo', 'siprop': 'namespaces'} try: - data = query.GetData(predata, site = site, useAPI = True)['query']['namespaces'] + data = query.GetData(predata, site)['query']['namespaces'] except wikipedia.ServerError, e: wikipedia.output(u'Warning! %s: %s' % (site, e)) return Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/welcome.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -506,8 +506,7 @@ 'usprop' :'blockinfo', } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) # If there's not the blockedby parameter (that means the user isn't blocked), it will return False otherwise True. try: blockedBy = data['query']['users'][0]['blockedby'] Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/wikipedia.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -919,8 +919,7 @@ 'prop' :'info', 'titles' :self.title(), } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) pageid = data['query']['pages'].keys()[0] if data['query']['pages'][pageid].keys()[0] == 'lastrevid': return data['query']['pages'][pageid]['lastrevid'] # if ok, @@ -952,8 +951,7 @@ 'tllimit' :tllimit, } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: pageid = data['query']['pages'].keys()[0] except KeyError: @@ -1327,7 +1325,7 @@ #if titles: # predata['titles'] = query.ListToParam(titles) - text = query.GetData(predata, useAPI = True)['query']['pages'] + text = query.GetData(predata)['query']['pages'] for pageid in text: if text[pageid].has_key('missing'): @@ -2765,7 +2763,7 @@ 'titles' :self.title(), } try: - data = query.GetData(params, useAPI = True, encodeTitle = False)['query']['pages'] + data = query.GetData(params, encodeTitle = False)['query']['pages'] except KeyError: raise NoPage(u'API Error, nothing found in the APIs') @@ -2829,7 +2827,7 @@ 'titles' :self.title(), 'iiprop' :'url', } - imagedata = query.GetData(params, useAPI = True, encodeTitle = False) + imagedata = query.GetData(params, encodeTitle = False) try: url=imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] # urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) @@ -2899,7 +2897,7 @@ 'prop' :'imageinfo', 'titles' :self.title(), } - data = query.GetData(params, useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: # We don't know the page's id, if any other better idea please change it pageid = data['query']['pages'].keys()[0] @@ -2921,7 +2919,7 @@ 'iiprop' :'sha1', } # First of all we need the Hash that identify an image - data = query.GetData(params, useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) pageid = data['query']['pages'].keys()[0] try: hash_found = data['query']['pages'][pageid][u'imageinfo'][0][u'sha1'] @@ -4505,7 +4503,7 @@ 'meta': 'userinfo', 'uiprop': 'blockinfo', } - data = query.GetData(params, self, useAPI = True)['query']['userinfo'] + data = query.GetData(params, self)['query']['userinfo'] return data.has_key('blockby') except NotImplementedError: return False @@ -5121,7 +5119,7 @@ 'meta':'allmessages', } try: - datas = query.GetData(params, useAPI = True)['query']['allmessages'] + datas = query.GetData(params)['query']['allmessages'] except KeyError: raise ServerError("The APIs don't return data, the site may be down") except NotImplementedError: @@ -5222,7 +5220,7 @@ 'meta': 'userinfo', 'uiprop': 'blockinfo|groups|rights|hasmsg|ratelimits|preferencestoken', } - text = query.GetData(params, site = self, useAPI = True, sysop=sysop)['query']['userinfo'] + text = query.GetData(params, self, sysop=sysop)['query']['userinfo'] ##output('%s' % text) # for debug use only else: url = self.edit_address('Non-existing_page') @@ -5503,8 +5501,7 @@ if leuser is not None: params['leuser'] = leuser if letitle is not None: params['letitle'] = letitle while True: - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: imagesData = data['query']['logevents'] except KeyError: @@ -5572,8 +5569,7 @@ if rcshow is not None: params['rcshow'] = rcshow if rctype is not None: params['rctype'] = rctype while True: - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: rcData = data['query']['recentchanges'] except KeyError: @@ -5709,7 +5705,7 @@ 'rnlimit': '1', #'': '', } - data = query.GetData(params, useAPI = True) + data = query.GetData(params) return Page(self, data['query']['random'][0]['title']) else: """Yield random page via Special:Random""" @@ -5727,7 +5723,7 @@ 'rnlimit': '1', 'rnredirect': '1', } - data = query.GetData(params, useAPI = True) + data = query.GetData(params) return Page(self, data['query']['random'][0]['title']) else: """Yield random redirect page via Special:RandomRedirect.""" @@ -5781,7 +5777,7 @@ params['apfrom'] = start if throttle: get_throttle() - data = query.GetData(params, useAPI = True) + data = query.GetData(params) for p in data['query']['allpages']: yield Page(self, p['title']) @@ -5973,7 +5969,7 @@ 'euquery': url, } while True: - data = query.GetData(params, useAPI = True) + data = query.GetData(params) if data['query']['exturlusage'] == []: break for pages in data['query']['exturlusage']: @@ -6586,7 +6582,7 @@ 'list' :'allimages', 'aisha1' :hash_found, } - allimages = query.GetData(params, site = getSite(self.lang, self.family), useAPI = True, encodeTitle = False)['query']['allimages'] + allimages = query.GetData(params, getSite(self.lang, self.family), encodeTitle = False)['query']['allimages'] files = list() for imagedata in allimages: image = imagedata[u'name']

1 0

SVN: [7113] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7113 Author: alexsh Date: 2009-08-05 21:39:21 +0000 (Wed, 05 Aug 2009) Log Message: ----------- query.py: set useAPI default True..(query.php is removed for long times, not necessary to set it default) Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-05 21:25:39 UTC (rev 7112) +++ trunk/pywikipedia/query.py 2009-08-05 21:39:21 UTC (rev 7113) @@ -14,8 +14,7 @@ 'rvprop' :'user|timestamp|content', } - print query.GetData(params, - useAPI = True, encodeTitle = False) + print query.GetData(params, encodeTitle = False) """ # @@ -28,7 +27,7 @@ import wikipedia, simplejson, urllib, time -def GetData(params, site = None, verbose = False, useAPI = False, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): +def GetData(params, site = None, verbose = False, useAPI = True, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): """Get data from the query api, and convert it into a data object """ if site is None:

1 0

SVN: [7112] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7112 Author: alexsh Date: 2009-08-05 21:25:39 +0000 (Wed, 05 Aug 2009) Log Message: ----------- Site()._load() and Site()._getUserData(): for reduce load time, Add query API option to collect user data. (use ac=query,meta=userinfo) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-05 06:24:39 UTC (rev 7111) +++ trunk/pywikipedia/wikipedia.py 2009-08-05 21:25:39 UTC (rev 7112) @@ -1499,6 +1499,7 @@ predata['wpWatchthis'] = '1' # Give the token, but only if one is supplied. if token: + ##output(token) # for debug use only predata['wpEditToken'] = token # Sorry, single-site exception... @@ -1630,6 +1631,7 @@ except NoUsername: raise LockedPage() if not newToken and "<textarea" in data: + ##if "<textarea" in data: # for debug use only, if badtoken still happen # We might have been using an outdated token output(u"Changing page has failed. Retrying.") return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop, getagain = True), newToken = True, sysop = sysop) @@ -4519,6 +4521,7 @@ else: self._load(sysop = sysop) index = self._userIndex(sysop) + ##output('%s' % self._rights[index]) #for debug use return right in self._rights[index] def server_time(self): @@ -4874,126 +4877,209 @@ * text - the page text * sysop - is the user a sysop? """ - if '<div id="globalWrapper">' not in text: - # Not a wiki page - return index = self._userIndex(sysop) - # Check for blocks - but only if version is 1.11 (userinfo is available) - # and the user data was not yet loaded - if self.versionnumber() >= 11 and (not self._userData[index] or force): - blocked = self._getBlock(sysop = sysop) - if blocked and not self._isBlocked[index]: + if type(text) == dict: #text is dict, query from API + # Check for blocks - but only if version is 1.11 (userinfo is available) + # and the user data was not yet loaded + if text.has_key('blockedby') and not self._isBlocked[index]: # Write a warning if not shown earlier if sysop: account = 'Your sysop account' else: account = 'Your account' output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) - self._isBlocked[index] = blocked + self._isBlocked[index] = text.has_key('blockedby') - # Check for new messages - if '<div class="usermessage">' in text: - if not self._messages[index]: - # User has *new* messages - if sysop: - output(u'NOTE: You have new messages in your sysop account on %s' % self) - else: - output(u'NOTE: You have new messages on %s' % self) - self._messages[index] = True - else: - self._messages[index] = False + # Check for new messages, show key 'messages' in dict. + if text.has_key('messages'): + if not self._messages[index]: + # User has *new* messages + if sysop: + output(u'NOTE: You have new messages in your sysop account on %s' % self) + else: + output(u'NOTE: You have new messages on %s' % self) + self._messages[index] = True + else: + self._messages[index] = False - # Don't perform other checks if the data was already loaded - if self._userData[index] and not force: - return + # Don't perform other checks if the data was already loaded + if self._userData[index] and not force: + return - # Search for the the user page link at the top. - # Note that the link of anonymous users (which doesn't exist at all - # in Wikimedia sites) has the ID pt-anonuserpage, and thus won't be - # found here. - userpageR = re.compile('<li id="pt-userpage"><a href=".+?">(?P<username>.+?)</a></li>') - m = userpageR.search(text) - if m: - self._isLoggedIn[index] = True - self._userName[index] = m.group('username') - else: - self._isLoggedIn[index] = False - # No idea what is the user name, and it isn't important - self._userName[index] = None + # Get username. + # anonymous mode will show key 'anon' + if not text.has_key('anon'): + self._isLoggedIn[index] = True + self._userName[index] = text['name'] + else: + self._isLoggedIn[index] = False + # No idea what is the user name, and it isn't important + self._userName[index] = None - # Check user groups, if possible (introduced in 1.10) - groupsR = re.compile(r'var wgUserGroups = \[\"(.+)\"\];') - m = groupsR.search(text) - checkLocal = True - if default_code in self.family.cross_allowed: # if current languages in cross allowed list, check global bot flag. - globalgroupsR = re.compile(r'var wgGlobalGroups = \[\"(.+)\"\];') - mg = globalgroupsR.search(text) - if mg: # the account had global permission - globalRights = mg.group(1) - globalRights = globalRights.split('","') - self._rights[index] = globalRights + # Check user groups and rights + if text.has_key('groups') and text['groups'] != []: + self._rights[index] = text['groups'] + self._rights[index].extend(text['rights']) + # Warnings + # Don't show warnings for not logged in users, they will just fail to + # do any action if self._isLoggedIn[index]: - if 'Global_bot' in globalRights: # This account has the global bot flag, no need to check local flags. - checkLocal = False - else: - output(u'Your bot account does not have global the bot flag, checking local flag.') + if 'bot' not in self._rights[index] and config.notify_unflagged_bot: + # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + if sysop: + output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + else: + output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) + if sysop and 'sysop' not in self._rights[index]: + output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) + else: + # key groups is not exists, setup a default rights + self._rights[index] = [] + if self._isLoggedIn[index]: + # Logged in user + self._rights[index].append('user') + # Assume bot, and thus autoconfirmed + self._rights[index].extend(['bot', 'autoconfirmed']) + if sysop: + # Assume user reported as a sysop indeed has the sysop rights + self._rights[index].append('sysop') + # Assume the user has the default rights if API not query back + self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) + #remove Duplicate rights + self._rights[index] = list(set(self._rights[index])) + + # Search for a token + if text.has_key('preferencestoken') and len(text['preferencestoken']) > 2: + # anonymous token is '+\\', check len('+\\') = 2 + # if preferencestoken > 2, it must be loggedin. + self._token[index] = text['preferencestoken'] + if self._rights[index] is not None: + # In this case, token and rights are loaded - user data is now loaded + self._userData[index] = True + else: + output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) else: - if verbose: output(u'Note: this language does not allow global bots.') - if m and checkLocal: - rights = m.group(1) - rights = rights.split('", "') - if '*' in rights: - rights.remove('*') - self._rights[index] = rights - # Warnings - # Don't show warnings for not logged in users, they will just fail to - # do any action - if self._isLoggedIn[index]: - if 'bot' not in self._rights[index] and config.notify_unflagged_bot: - # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + #ordinary mode to get data from edit page HTMLs and JavaScripts + + if '<div id="globalWrapper">' not in text: + # Not a wiki page + return + # Check for blocks - but only if version is 1.11 (userinfo is available) + # and the user data was not yet loaded + if self.versionnumber() >= 11 and (not self._userData[index] or force): + blocked = self._getBlock(sysop = sysop) + if blocked and not self._isBlocked[index]: + # Write a warning if not shown earlier if sysop: - output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + account = 'Your sysop account' else: - output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) - if sysop and 'sysop' not in self._rights[index]: - output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) - else: - # We don't have wgUserGroups, and can't check the rights - self._rights[index] = [] - if self._isLoggedIn[index]: - # Logged in user - self._rights[index].append('user') - # Assume bot, and thus autoconfirmed - self._rights[index].extend(['bot', 'autoconfirmed']) - if sysop: - # Assume user reported as a sysop indeed has the sysop rights - self._rights[index].append('sysop') - # Assume the user has the default rights - self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) - if 'bot' in self._rights[index] or 'sysop' in self._rights[index]: - self._rights[index].append('apihighlimits') - if 'sysop' in self._rights[index]: - self._rights[index].extend(['delete', 'undelete', 'block', 'protect', 'import', 'deletedhistory', 'unwatchedpages']) + account = 'Your account' + output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) + self._isBlocked[index] = blocked - # Search for a token - tokenR = re.compile(r"\<input type='hidden' value=\"(.*?)\" name=\"wpEditToken\"") - tokenloc = tokenR.search(text) - if tokenloc: - self._token[index] = tokenloc.group(1) - if self._rights[index] is not None: - # In this case, token and rights are loaded - user data is now loaded - self._userData[index] = True - else: - # Token not found - # Possible reason for this is the user is blocked, don't show a - # warning in this case, otherwise do show a warning - # Another possible reason is that the page cannot be edited - ensure - # there is a textarea and the tab "view source" is not shown - if u'<textarea' in text and u'<li id="ca-viewsource"' not in text and not self._isBlocked[index]: + # Check for new messages + if '<div class="usermessage">' in text: + if not self._messages[index]: + # User has *new* messages + if sysop: + output(u'NOTE: You have new messages in your sysop account on %s' % self) + else: + output(u'NOTE: You have new messages on %s' % self) + self._messages[index] = True + else: + self._messages[index] = False + + # Don't perform other checks if the data was already loaded + if self._userData[index] and not force: + return + + # Search for the the user page link at the top. + # Note that the link of anonymous users (which doesn't exist at all + # in Wikimedia sites) has the ID pt-anonuserpage, and thus won't be + # found here. + userpageR = re.compile('<li id="pt-userpage"><a href=".+?">(?P<username>.+?)</a></li>') + m = userpageR.search(text) + if m: + self._isLoggedIn[index] = True + self._userName[index] = m.group('username') + else: + self._isLoggedIn[index] = False + # No idea what is the user name, and it isn't important + self._userName[index] = None + + # Check user groups, if possible (introduced in 1.10) + groupsR = re.compile(r'var wgUserGroups = \[\"(.+)\"\];') + m = groupsR.search(text) + checkLocal = True + if default_code in self.family.cross_allowed: # if current languages in cross allowed list, check global bot flag. + globalgroupsR = re.compile(r'var wgGlobalGroups = \[\"(.+)\"\];') + mg = globalgroupsR.search(text) + if mg: # the account had global permission + globalRights = mg.group(1) + globalRights = globalRights.split('","') + self._rights[index] = globalRights + if self._isLoggedIn[index]: + if 'Global_bot' in globalRights: # This account has the global bot flag, no need to check local flags. + checkLocal = False + else: + output(u'Your bot account does not have global the bot flag, checking local flag.') + else: + if verbose: output(u'Note: this language does not allow global bots.') + if m and checkLocal: + rights = m.group(1) + rights = rights.split('", "') + if '*' in rights: + rights.remove('*') + self._rights[index] = rights + # Warnings + # Don't show warnings for not logged in users, they will just fail to + # do any action + if self._isLoggedIn[index]: + if 'bot' not in self._rights[index] and config.notify_unflagged_bot: + # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + if sysop: + output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + else: + output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) + if sysop and 'sysop' not in self._rights[index]: + output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) + else: + # We don't have wgUserGroups, and can't check the rights + self._rights[index] = [] + if self._isLoggedIn[index]: + # Logged in user + self._rights[index].append('user') + # Assume bot, and thus autoconfirmed + self._rights[index].extend(['bot', 'autoconfirmed']) + if sysop: + # Assume user reported as a sysop indeed has the sysop rights + self._rights[index].append('sysop') + # Assume the user has the default rights + self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) + if 'bot' in self._rights[index] or 'sysop' in self._rights[index]: + self._rights[index].append('apihighlimits') + if 'sysop' in self._rights[index]: + self._rights[index].extend(['delete', 'undelete', 'block', 'protect', 'import', 'deletedhistory', 'unwatchedpages']) + + # Search for a token + tokenR = re.compile(r"\<input type='hidden' value=\"(.*?)\" name=\"wpEditToken\"") + tokenloc = tokenR.search(text) + if tokenloc: + self._token[index] = tokenloc.group(1) + if self._rights[index] is not None: + # In this case, token and rights are loaded - user data is now loaded + self._userData[index] = True + else: # Token not found - output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) + # Possible reason for this is the user is blocked, don't show a + # warning in this case, otherwise do show a warning + # Another possible reason is that the page cannot be edited - ensure + # there is a textarea and the tab "view source" is not shown + if u'<textarea' in text and u'<li id="ca-viewsource"' not in text and not self._isBlocked[index]: + # Token not found + output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) def mediawiki_message(self, key): """Return the MediaWiki message text for key "key" """ @@ -5121,10 +5207,26 @@ if verbose: output(u'Getting information for site %s' % self) - + + try: + api_url = self.api_address() + del api_url + except NotImplementedError: + config.use_api = False + # Get data - url = self.edit_address('Non-existing_page') - text = self.getUrl(url, sysop = sysop) + if config.use_api and self.versionnumber() >= 11: + #Query userinfo + params = { + 'action': 'query', + 'meta': 'userinfo', + 'uiprop': 'blockinfo|groups|rights|hasmsg|ratelimits|preferencestoken', + } + text = query.GetData(params, site = self, useAPI = True, sysop=sysop)['query']['userinfo'] + ##output('%s' % text) # for debug use only + else: + url = self.edit_address('Non-existing_page') + text = self.getUrl(url, sysop = sysop) # Parse data self._getUserData(text, sysop = sysop, force = force) @@ -5862,7 +5964,7 @@ if not siteurl.startswith('*.'): urlsToRetrieve.append('*.' + siteurl) if config.use_api: - output(u'Querying API...') + output(u'Querying API exturlusage...') for url in urlsToRetrieve: params = { 'action': 'query',

1 0

SVN: [7111] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7111 Author: alexsh Date: 2009-08-05 06:24:39 +0000 (Wed, 05 Aug 2009) Log Message: ----------- query.GetData(): add sysop parameter for handle cookie in postData and getUrl Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-04 21:29:45 UTC (rev 7110) +++ trunk/pywikipedia/query.py 2009-08-05 06:24:39 UTC (rev 7111) @@ -28,7 +28,7 @@ import wikipedia, simplejson, urllib, time -def GetData(params, site = None, verbose = False, useAPI = False, retryCount = 5, encodeTitle = True, back_response = False): +def GetData(params, site = None, verbose = False, useAPI = False, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): """Get data from the query api, and convert it into a data object """ if site is None: @@ -82,12 +82,12 @@ try: jsontext = "Nothing received" if params['action'] in postAC: - res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies()) + res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(), sysop=sysop) else: if back_response: - res, jsontext = site.getUrl( path, retry=True, data=data, back_response=True) + res, jsontext = site.getUrl( path, retry=True, data=data, sysop=sysop, back_response=True) else: - jsontext = site.getUrl( path, retry=True, data=data) + jsontext = site.getUrl( path, retry=True, sysop=sysop, data=data) # This will also work, but all unicode strings will need to be converted from \u notation # decodedObj = eval( jsontext )

1 0

SVN: [7110] trunk/pywikipedia/maintenance/update_namespaces.py
by alexsh＠svn.wikimedia.org 04 Aug '09

04 Aug '09

Revision: 7110 Author: alexsh Date: 2009-08-04 21:29:45 +0000 (Tue, 04 Aug 2009) Log Message: ----------- disable update main family in default way, remove -nomain and add -upmain to enable it Modified Paths: -------------- trunk/pywikipedia/maintenance/update_namespaces.py Modified: trunk/pywikipedia/maintenance/update_namespaces.py =================================================================== --- trunk/pywikipedia/maintenance/update_namespaces.py 2009-08-04 21:28:28 UTC (rev 7109) +++ trunk/pywikipedia/maintenance/update_namespaces.py 2009-08-04 21:29:45 UTC (rev 7110) @@ -3,7 +3,7 @@ both the generic family.py and the site-specific family. options: - -nomain Don't modify the main family.py + -upmain Modify the main family.py, too. -wikimedia Update all the wikimedia families """ # @@ -103,11 +103,11 @@ if __name__ == '__main__': try: - update_main_family = True + update_main_family = False update_wikimedia = False for arg in wikipedia.handleArgs(): - if arg == '-nomain': - update_main_family = False + if arg == '-upmain': + update_main_family = True elif arg == '-wikimedia': update_wikimedia = True

1 0

SVN: [7109] trunk/pywikipedia/speedy_delete.py
by alexsh＠svn.wikimedia.org 04 Aug '09

04 Aug '09

Revision: 7109 Author: alexsh Date: 2009-08-04 21:28:28 +0000 (Tue, 04 Aug 2009) Log Message: ----------- update zh CSD reason Modified Paths: -------------- trunk/pywikipedia/speedy_delete.py Modified: trunk/pywikipedia/speedy_delete.py =================================================================== --- trunk/pywikipedia/speedy_delete.py 2009-08-04 20:41:16 UTC (rev 7108) +++ trunk/pywikipedia/speedy_delete.py 2009-08-04 21:28:28 UTC (rev 7109) @@ -398,7 +398,7 @@ 'test':u'[[WP:CSD#G2]]: 測試頁', 'vand':u'[[WP:CSD#G3]]: 純粹破壞', 'cont':u'[[WP:CSD#G4]]: 非常短，而且沒有定義或內容。', - 'rep':u'[[WP:CSD#G5]]: 重新建立的內容', + 'rep':u'[[WP:CSD#G5]]: 重複的條目或是重新建立的內容', 'oprj':u'[[WP:CSD#G7]]: 內容來自其他中文計劃', 'text':u'[[WP:CSD#G9]]: 只有相關連結、項目的頁面', 'auth':u'[[WP:CSD#G10]]: 原作者請求',

1 0

SVN: [7108] trunk/pywikipedia/wikipedia.py
by erwin85＠svn.wikimedia.org 04 Aug '09

04 Aug '09

Revision: 7108 Author: erwin85 Date: 2009-08-04 20:41:16 +0000 (Tue, 04 Aug 2009) Log Message: ----------- Fix getFileVersionHistory Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-04 08:20:10 UTC (rev 7107) +++ trunk/pywikipedia/wikipedia.py 2009-08-04 20:41:16 UTC (rev 7108) @@ -2868,7 +2868,7 @@ """ result = [] - history = re.search('(?s)<table class="filehistory">.+?</table>', self.getImagePageHtml()) + history = re.search('(?s)<table class="wikitable filehistory">.+?</table>', self.getImagePageHtml()) if history: lineR = re.compile(r'<tr>(?:<td>.*?</td>){1,2}<td.*?><a href=".+?">(?P<datetime>.+?)</a></td><td>.*?(?P<resolution>\d+\xd7\d+) <span.*?>$(?P<filesize>.+?)$</span></td><td><a href=".+?"(?: class="new"|) title=".+?">(?P<username>.+?)</a>.*?</td><td>(?:.*?<span class="comment">$(?P<comment>.*?)$</span>)?</td></tr>') if not lineR.search(history.group()):

1 0

SVN: [7107] trunk/pywikipedia/families/wikipedia_family.py
by valhallasw＠svn.wikimedia.org 04 Aug '09

04 Aug '09

Revision: 7107 Author: valhallasw Date: 2009-08-04 08:20:10 +0000 (Tue, 04 Aug 2009) Log Message: ----------- Updated crossnamespace to ar, per bug #2822705 Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-04 06:27:42 UTC (rev 7106) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-04 08:20:10 UTC (rev 7107) @@ -979,6 +979,7 @@ '_default': { 'pt': [102], 'als': [104], + 'ar': [104], 'en': [12], 'es': [104], 'fi': [4],

1 0

SVN: [7106] trunk/pywikipedia/pagegenerators.py
by purodha＠svn.wikimedia.org 04 Aug '09

04 Aug '09

Revision: 7106 Author: purodha Date: 2009-08-04 06:27:42 +0000 (Tue, 04 Aug 2009) Log Message: ----------- index was off by one. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2009-08-04 02:00:14 UTC (rev 7105) +++ trunk/pywikipedia/pagegenerators.py 2009-08-04 06:27:42 UTC (rev 7106) @@ -1075,7 +1075,7 @@ gen = AllpagesPageGenerator(firstPageTitle, namespace, includeredirects=False) elif arg.startswith('-redirectonly'): - firstPageTitle = arg[15:] + firstPageTitle = arg[14:] if not firstPageTitle: firstPageTitle = wikipedia.input( u'At which page do you want to start?')

1 0

SVN: [7105] trunk/pywikipedia/family.py
by valhallasw＠svn.wikimedia.org 03 Aug '09

03 Aug '09

Revision: 7105 Author: valhallasw Date: 2009-08-04 02:00:14 +0000 (Tue, 04 Aug 2009) Log Message: ----------- Updated redirect for qu (fixes bug #2829501, based on MessagesQu.php r54295) Modified Paths: -------------- trunk/pywikipedia/family.py Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2009-08-04 01:49:38 UTC (rev 7104) +++ trunk/pywikipedia/family.py 2009-08-04 02:00:14 UTC (rev 7105) @@ -3445,7 +3445,7 @@ 'pl': [u'patrz', u'przekieruj', u'tam'], 'pms': [u'rinvia', u'rinvio'], 'pt-br': [u'redirecionamento'], - 'qu': [u'redirección'], + 'qu': [u'pusapuna', u'redirección'], 'rmy': [u'redirecteaza'], 'ro': [u'redirecteaza'], 'ru': [u'перенаправление', u'перенапр'],

1 0

SVN: [7104] trunk/pywikipedia/pagegenerators.py
by valhallasw＠svn.wikimedia.org 03 Aug '09

03 Aug '09

Revision: 7104 Author: valhallasw Date: 2009-08-04 01:49:38 +0000 (Tue, 04 Aug 2009) Log Message: ----------- (Ugly) fix for bug #2831667. See http://www.mail-archive.com/python-dev@python.org/msg12668.html and follow-ups for more information. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2009-08-04 01:34:43 UTC (rev 7103) +++ trunk/pywikipedia/pagegenerators.py 2009-08-04 01:49:38 UTC (rev 7104) @@ -164,7 +164,14 @@ import wikipedia, date, catlib import config +# For python 2.4 compatibility +# see http://www.mail-archive.com/python-dev@python.org/msg12668.html +try: + GeneratorExit +except NameError: + class GeneratorExit(Exception): pass + class ThreadedGenerator(threading.Thread): """Look-ahead generator class.

1 0

SVN: [7103] trunk/pywikipedia/fixes.py
by valhallasw＠svn.wikimedia.org 03 Aug '09

03 Aug '09

Revision: 7103 Author: valhallasw Date: 2009-08-04 01:34:43 +0000 (Tue, 04 Aug 2009) Log Message: ----------- Fix bug introduced in r6945 (non-ascii characters should only show up in unicode strings). Fixes bug #2831669 Modified Paths: -------------- trunk/pywikipedia/fixes.py Modified: trunk/pywikipedia/fixes.py =================================================================== --- trunk/pywikipedia/fixes.py 2009-08-02 12:05:24 UTC (rev 7102) +++ trunk/pywikipedia/fixes.py 2009-08-04 01:34:43 UTC (rev 7103) @@ -2,7 +2,7 @@ """ File containing all standard fixes """ __version__ = '$Id$' -help = """ +help = u""" * HTML - Convert HTML tags to wiki syntax, and fix XHTML. * isbn - Fix badly formatted ISBNs.

1 0

SVN: [7102] trunk/pywikipedia/category.py
by valhallasw＠svn.wikimedia.org 02 Aug '09

02 Aug '09

Revision: 7102 Author: valhallasw Date: 2009-08-02 12:05:24 +0000 (Sun, 02 Aug 2009) Log Message: ----------- Fixed incorrect translations (using wrong number of %s-es) * by siebrand (r6285, nl translation) (fixed myself) * by alexsh (7015, cs translation) (thanks to egg from #wikipedia_cs IRC) Modified Paths: -------------- trunk/pywikipedia/category.py Modified: trunk/pywikipedia/category.py =================================================================== --- trunk/pywikipedia/category.py 2009-07-27 15:54:13 UTC (rev 7101) +++ trunk/pywikipedia/category.py 2009-08-02 12:05:24 UTC (rev 7102) @@ -94,7 +94,7 @@ 'bat-smg':u'Robots: Pridedama [[Kateguorėjė:%s]]', 'be-x-old':u'Робат: дадаваньне [[Катэгорыя:%s]]', 'ca':u'Robot: Afegint [[Categoria:%s]]', - 'cs':u'Robot přidal kategorii [[Kategorie:%s|%s]]', + 'cs':u'Robot přidal [[Kategorie:%s]]', 'da':u'Robot: Tilføjer [[Kategori:%s]]', 'de':u'Bot: Ergänze [[Kategorie:%s]]', 'en':u'Robot: Adding [[Category:%s]]', @@ -114,7 +114,7 @@ 'lt':u'robotas: Pridedama [[Kategorija:%s]]', 'nds':u'Kat-Bot: [[Kategorie:%s]] rin', 'nds-nl':u'bot: [[kattegerie:%s]] derbie edaon', - 'nl':u'Bot: [[Categorie:%s|categorie %s]] toegevoegd', + 'nl':u'Bot: [[categorie:%s]] toegevoegd', 'no':u'Robot: Legger til [[Kategori:%s]]', 'nn':u'robot: la til [[Kategori:%s]]', 'pl':u'Robot dodaje [[Kategoria:%s]]', @@ -165,7 +165,7 @@ 'bat-smg':u'Robots: Kateguorėjė bova parvadėnta i [[:Kateguorėjė:%s|%s]]', 'be-x-old':u'Робат: катэгорыя перайменаваная ў [[:Катэгорыя:%s|%s]]', 'ca':u'Robot: La categoria s\'ha mogut a [[:Categoria:%s|%s]]', - 'cs':u'Kategorie přesunuta na %s', + 'cs':u'Kategorie přesunuta na [[:Kategorie:%s|%s]]', 'da':u'Robot: Kategori flyttet til [[:Category:%s|%s]]', 'de':u'Bot: Kategorie wurde nach [[:Category:%s|%s]] verschoben', 'en':u'Robot: Category was moved to [[:Category:%s|%s]]',

1 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn August 2009