Revision: 5826 Author: multichill Date: 2008-08-21 14:26:24 +0000 (Thu, 21 Aug 2008) Log Message: ----------- Country filter and option to only filter. Modified Paths: -------------- trunk/pywikipedia/imagerecat.py Modified: trunk/pywikipedia/imagerecat.py =================================================================== --- trunk/pywikipedia/imagerecat.py 2008-08-21 13:23:10 UTC (rev 5825) +++ trunk/pywikipedia/imagerecat.py 2008-08-21 14:26:24 UTC (rev 5826) @@ -23,23 +23,41 @@ category_blacklist = [u'Hidden categories', u'Stub pictures'] -def categorizeImages(generator): +countries = [] + +def getCountries(): + ''' + Get the list of countries from Commons. + ''' + result = [] + countryPage = wikipedia.Page(wikipedia.getSite(), u'User:Multichill/Countries') + for country in countryPage.linkedPages(): + result.append(country.titleWithoutNamespace()) + return result + +def categorizeImages(generator, onlyfilter): + ''' + Loop over all images in generator and try to categorize them. Get category suggestions from CommonSense. + ''' for page in generator: if page.exists() and (page.namespace() == 6) and (not page.isRedirectPage()): imagepage = wikipedia.ImagePage(page.site(), page.title()) #imagepage.get() wikipedia.output(u'Working on ' + imagepage.title()); currentCats = getCurrentCats(imagepage) - commonshelperCats = getCommonshelperCats(imagepage) + if(onlyfilter): + commonshelperCats = [] + else: + commonshelperCats = getCommonshelperCats(imagepage) newcats = filterBlacklist(commonshelperCats+currentCats) newcats = filterDisambiguation(newcats) newcats = followRedirects(newcats) - #newcats = filterCountries(newcats) + newcats = filterCountries(newcats) newcats = filterParents(newcats) - if len(newcats) > 0: + if (len(newcats) > 0 and not(set(currentCats)==set(newcats))): for cat in newcats: wikipedia.output(u' Found new cat: ' + cat); - saveImagePage(imagepage, newcats) + saveImagePage(imagepage, newcats, onlyfilter) def getCurrentCats(imagepage): @@ -54,7 +72,7 @@ def getCommonshelperCats(imagepage): ''' - Get category suggestions from commonshelper. Parse them and return a list of suggestions. + Get category suggestions from CommonSense. Parse them and return a list of suggestions. ''' result = [] parameters = urllib.urlencode({'i' : imagepage.titleWithoutNamespace().encode('utf-8'), 'r' : 'on', 'go-clean' : 'Find+Categories', 'cl' : 'li'}) @@ -82,6 +100,9 @@ def filterBlacklist(categories): + ''' + Filter out categories which are on the blacklist. + ''' result = [] for cat in categories: if (cat not in category_blacklist): @@ -90,6 +111,9 @@ def filterDisambiguation(categories): + ''' + Filter out disambiguation categories. + ''' result = [] for cat in categories: if(not wikipedia.Page(wikipedia.getSite(), u'Category:' + cat).isDisambig()): @@ -97,6 +121,9 @@ return result def followRedirects(categories): + ''' + If a category is a redirect, replace the category with the target. + ''' result = [] for cat in categories: categoryPage = wikipedia.Page(wikipedia.getSite(), u'Category:' + cat) @@ -110,13 +137,40 @@ def filterCountries(categories): - result = [] - return result + ''' + Try to filter out ...by country categories. + First make a list of any ...by country categories and try to find some countries. + If a by country category has a subcategoy containing one of the countries found, add it. + The ...by country categories remain in the set and should be filtered out by filterParents. + ''' + result = categories + listByCountry = [] + listCountries = [] + for cat in categories: + if (cat.endswith(u'by country')): + listByCountry.append(cat) + + #If cat contains 'by country' add it to the list + #If cat contains the name of a country add it to the list + else: + for country in countries: + if not(cat.find(country)==-1): + listCountries.append(country) + + if(len(listByCountry) > 0): + for bc in listByCountry: + category = catlib.Category(wikipedia.getSite(), u'Category:' + bc) + for subcategory in category.subcategories(): + for country in listCountries: + if (subcategory.titleWithoutNamespace().endswith(country)): + result.append(subcategory.titleWithoutNamespace()) + + return list(set(result)) def filterParents(categories): ''' - Remove the current categories from the suggestions and remove blacklisted cats. + Remove all parent categories from the set to prevent overcategorization. ''' result = [] toFilter = u'' @@ -135,18 +189,29 @@ return result -def saveImagePage(imagepage, newcats): +def saveImagePage(imagepage, newcats, onlyfilter): + ''' + Remove the old categories and add the new categories to the image. + ''' newtext = wikipedia.removeCategoryLinks(imagepage.get(), imagepage.site()) newtext = removeTemplates(newtext) + u'{{subst:chc}}\n' for category in newcats: newtext = newtext + u'[[Category:' + category + u']]\n' + + if(onlyfilter): + comment = u'Filtering categories' + else: + comment = u'Image is categorized by a bot using data from [[Commons:Tools#CommonSense|CommonSense]]' wikipedia.showDiff(imagepage.get(), newtext) - imagepage.put(newtext, u'Image is categorized by a bot using data from [[Commons:Tools#CommonSense|CommonSense]]') + imagepage.put(newtext, comment) return def removeTemplates(oldtext = u''): + ''' + Remove {{Uncategorized}} and {{Check categories}} templates + ''' result = u'' result = re.sub(u'\{\{\s*([Uu]ncat(egori[sz]ed( image)?)?|[Nn]ocat|[Nn]eedscategory)[^}]*\}\}', u'', oldtext) result = re.sub(u'<!-- Remove this line once you have added categories -->', u'', result) @@ -158,8 +223,8 @@ ''' Main loop. Get a generator. Set up the 3 threads and the 2 queue's and fire everything up. ''' - generator = None; - + generator = None + onlyfilter = False genFactory = pagegenerators.GeneratorFactory() site = wikipedia.getSite(u'commons', u'commons') @@ -170,12 +235,15 @@ generator = [wikipedia.Page(site, wikipedia.input(u'What page do you want to use?'))] else: generator = [wikipedia.Page(site, arg[6:])] + elif arg == '-onlyfilter': + onlyfilter = True else: generator = genFactory.handleArg(arg) if not generator: generator = pagegenerators.CategorizedPageGenerator(catlib.Category(site, u'Category:Media needing categories'), recurse=True) - - categorizeImages(generator) + global countries + countries = getCountries() + categorizeImages(generator, onlyfilter) wikipedia.output(u'All done')