jenkins-bot has submitted this change and it was merged.
Change subject: (bug 64087) fix for indexing set in category.py; code improvements
......................................................................
(bug 64087) fix for indexing set in category.py; code improvements
with https://www.mediawiki.org/wiki/Special:Code/pywikipedia/6428
a list was replaced by a set. Now indexing a set failes.
Solving this problem, we can easily replace that indexing by
enumerating the set which gives us both: the index value and the
category.
Some code improvements:
cat.title().split(':', 1)[1] can be replaced by cat.title(withNamespace=False)
Page.title() now is able to return labeled links if withNamespace parameter is False. e.g.
[:%s|%s]] % (sc[i].title(), sc[i]title().split(':', 1)[1]) can be replaced by a simple title() statement.
Adding "noisy dots" to show that bot is still running while
generating the category tree.
-depth option added for the maximum category depth. Default is
set to 5 which also needs a long time to compute it.
Change-Id: I463b7416bbf67ee8fb90dbeed525fc834aadc7d0
---
M scripts/category.py
1 file changed, 41 insertions(+), 32 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
Pyfisch: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/category.py b/scripts/category.py
index 81211c0..e12fd6c 100755
--- a/scripts/category.py
+++ b/scripts/category.py
@@ -63,6 +63,8 @@
and do not remove them
* -match - Only work on pages whose titles match the given regex (for
move and remove actions).
+ * -depth: - The max depth limit beyond which no subcategories will be
+ listed.
For the actions tidy and tree, the bot will store the category structure
locally in category.dump. This saves time and server load, but if it uses
@@ -747,22 +749,21 @@
pywikibot.output('\n' + full_text[:contextLength] + '\n')
- subcatlist = self.catDB.getSubcats(current_cat)
- supercatlist = self.catDB.getSupercats(current_cat)
+ # we need list to index the choice
+ subcatlist = list(self.catDB.getSubcats(current_cat))
+ supercatlist = list(self.catDB.getSupercats(current_cat))
- if len(subcatlist) == 0:
+ if not subcatlist:
pywikibot.output('This category has no subcategories.\n')
- if len(supercatlist) == 0:
+ if not supercatlist:
pywikibot.output('This category has no supercategories.\n')
# show subcategories as possible choices (with numbers)
- for i in range(len(supercatlist)):
+ for i, supercat in enumerate(supercatlist):
# layout: we don't expect a cat to have more than 10 supercats
- pywikibot.output(u'u%d - Move up to %s'
- % (i, supercatlist[i].title()))
- for i in range(len(subcatlist)):
+ pywikibot.output(u'u%d - Move up to %s' % (i, supercat.title()))
+ for i, subcat in enumerate(subcatlist):
# layout: we don't expect a cat to have more than 100 subcats
- pywikibot.output(u'%2d - Move down to %s'
- % (i, subcatlist[i].title()))
+ pywikibot.output(u'%2d - Move down to %s' % (i, subcat.title()))
pywikibot.output(' j - Jump to another category')
pywikibot.output(' s - Skip this article')
pywikibot.output(' r - Remove this category tag')
@@ -843,8 +844,7 @@
class CategoryTreeRobot:
- '''
- Robot to create tree overviews of the category structure.
+ """ Robot to create tree overviews of the category structure.
Parameters:
* catTitle - The category which will be the tree's root.
@@ -854,7 +854,7 @@
won't be a problem.
* filename - The textfile where the tree should be saved; None to print
the tree to stdout.
- '''
+ """
def __init__(self, catTitle, catDB, filename=None, maxDepth=10):
self.catTitle = catTitle
@@ -862,39 +862,41 @@
if filename and not os.path.isabs(filename):
filename = config.datafilepath(filename)
self.filename = filename
- # TODO: make maxDepth changeable with a parameter or config file entry
self.maxDepth = maxDepth
self.site = pywikibot.Site()
def treeview(self, cat, currentDepth=0, parent=None):
- '''
- Returns a multi-line string which contains a tree view of all
+ """ Return a multi-line string which contains a tree view of all
subcategories of cat, up to level maxDepth. Recursively calls itself.
Parameters:
* cat - the Category of the node we're currently opening
* currentDepth - the current level in the tree (for recursion)
* parent - the Category of the category we're coming from
- '''
- result = u'#' * currentDepth
- result += '[[:%s|%s]]' % (cat.title(), cat.title().split(':', 1)[1])
+ """
+
+ result = u'#' * currentDepth + ' '
+ result += cat.title(asLink=True, textlink=True, withNamespace=False)
result += ' (%d)' % len(self.catDB.getArticles(cat))
- # We will remove an element of this array, but will need the original
- # array later, so we create a shallow copy with [:]
- supercats = self.catDB.getSupercats(cat)[:]
+ # We will remove an element of supercats, but need the original set
+ # later, so we create a list from the catDB.getSupercats(cat) set
+ supercats = list(self.catDB.getSupercats(cat))
# Find out which other cats are supercats of the current cat
try:
supercats.remove(parent)
except:
pass
- if supercats != []:
+ if supercats:
+ if currentDepth < self.maxDepth / 2:
+ # noisy dots
+ pywikibot.output('.', newline=False)
supercat_names = []
- for i in range(len(supercats)):
+ for i, cat in enumerate(supercats):
# create a list of wiki links to the supercategories
- supercat_names.append('[[:%s|%s]]'
- % (supercats[i].title(),
- supercats[i].title().split(':', 1)[1]))
+ supercat_names.append(cat.title(asLink=True,
+ textlink=True,
+ withNamespace=False))
# print this list, separated with commas, using translations
# given in also_in_cats
result += ' ' + i18n.twtranslate(self.site, 'category-also-in',
@@ -906,9 +908,9 @@
# recurse into subdirectories
result += self.treeview(subcat, currentDepth + 1, parent=cat)
else:
- if self.catDB.getSubcats(cat) != []:
+ if self.catDB.getSubcats(cat):
# show that there are more categories beyond the depth limit
- result += '#' * (currentDepth + 1) + '[...]\n'
+ result += '#' * (currentDepth + 1) + ' [...]\n'
return result
def run(self):
@@ -921,7 +923,9 @@
"""
cat = pywikibot.Category(self.site, self.catTitle)
+ pywikibot.output('Generating tree...', newline=False)
tree = self.treeview(cat)
+ pywikibot.output(u'')
if self.filename:
pywikibot.output(u'Saving results in %s' % self.filename)
import codecs
@@ -947,6 +951,8 @@
titleRegex = None
pagesonly = False
withHistory = False
+ rebuild = False
+ depth = 5
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
@@ -958,7 +964,6 @@
# If this is set to true then the custom edit summary given for removing
# categories from articles will also be used as the deletion reason.
useSummaryForDeletion = True
- catDB = CategoryDatabase()
action = None
sort_by_last_name = False
create_pages = False
@@ -982,7 +987,7 @@
elif arg == '-person':
sort_by_last_name = True
elif arg == '-rebuild':
- catDB.rebuild()
+ rebuild = True
elif arg.startswith('-from:'):
oldCatTitle = arg[len('-from:'):].replace('_', ' ')
fromGiven = True
@@ -1019,9 +1024,13 @@
follow_redirects = True
elif arg == '-hist':
withHistory = True
+ elif arg.startswith('-depth:'):
+ depth = int(arg[len('-depth:'):])
else:
genFactory.handleArg(arg)
pywikibot.Site().login()
+
+ catDB = CategoryDatabase(rebuild=rebuild)
gen = genFactory.getCombinedGenerator()
if action == 'add':
if not gen:
@@ -1065,7 +1074,7 @@
filename = pywikibot.input(
u'Please enter the name of the file where the tree should be saved,'
u'\nor press enter to simply show the tree:')
- bot = CategoryTreeRobot(catTitle, catDB, filename)
+ bot = CategoryTreeRobot(catTitle, catDB, filename, depth)
bot.run()
elif action == 'listify':
if not fromGiven:
--
To view, visit https://gerrit.wikimedia.org/r/127251
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I463b7416bbf67ee8fb90dbeed525fc834aadc7d0
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Pyfisch <pyfisch(a)gmail.com>
Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)live.it>
Gerrit-Reviewer: Russell Blau <russblau(a)imapmail.org>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: add ImageGenerator and UnCategorizedTemplateGenerator
......................................................................
add ImageGenerator and UnCategorizedTemplateGenerator
ported from compat branch
APISite.uncategorizedtemplates() was available since
https://www.mediawiki.org/wiki/Special:Code/pywikipedia/9843
but had no corresponding pagegenerators
renamed UnCategorizedTemplatesGenerator
into UnCategorizedTemplateGenerator
for consistency reasons, as:
* UnCategorizedCategoryGenerator
* UnCategorizedImageGenerator
* UnCategorizedPageGenerator
bug: 55007
Change-Id: I961310f6b3f13f1ac3880857e0f454576a0cbce6
---
M pywikibot/pagegenerators.py
1 file changed, 17 insertions(+), 0 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index b33d3a3..ff64ab8 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -846,6 +846,16 @@
yield pywikibot.Category(page)
+def ImageGenerator(generator):
+ """
+ Wraps around another generator. Yields the same pages, but as ImagePage
+ objects instead of Page objects. Makes sense only if it is ascertained
+ that only images are being retrieved.
+ """
+ for page in generator:
+ yield pywikibot.ImagePage(page)
+
+
def PageWithTalkPageGenerator(generator):
"""Yield pages and associated talk pages from another generator.
@@ -983,6 +993,13 @@
yield page
+def UnCategorizedTemplateGenerator(total=100, site=None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.uncategorizedtemplates(total=total):
+ yield page
+
+
@deprecate_arg("number", "total")
@deprecate_arg("repeat", None)
def LonelyPagesPageGenerator(total=100, site=None):
--
To view, visit https://gerrit.wikimedia.org/r/126934
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I961310f6b3f13f1ac3880857e0f454576a0cbce6
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Ricordisamoa <ricordisamoa(a)live.it>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)live.it>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Allow site-specific, language-agnostic password file entries
......................................................................
Allow site-specific, language-agnostic password file entries
Allow defining entries in the password file that are specific
to a site but not to a language on that site.
Change-Id: I1390e0cc8ff72ee165bf115ef3c9d69e7d2f35af
---
M pywikibot/login.py
1 file changed, 15 insertions(+), 8 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/login.py b/pywikibot/login.py
index aead4be..43e8123 100644
--- a/pywikibot/login.py
+++ b/pywikibot/login.py
@@ -131,9 +131,11 @@
Use chmod 600 password-file.
All lines below should be valid Python tuples in the form
- (code, family, username, password) or (username, password)
- to set a default password for an username. Default usernames
- should occur above specific usernames.
+ (code, family, username, password),
+ (family, username, password) or
+ (username, password)
+ to set a default password for an username. The last matching entry will
+ be used, so default usernames should occur above specific usernames.
If the username or password contain non-ascii characters, they
should be stored using the utf-8 encoding.
@@ -142,21 +144,26 @@
(u"my_username", u"my_default_password")
(u"my_sysop_user", u"my_sysop_password")
- (u"en", u"wikipedia", u"my_en_user", u"my_en_pass")
+ (u"wikipedia", u"my_wikipedia_user", u"my_wikipedia_pass")
+ (u"en", u"wikipedia", u"my_en_wikipedia_user", u"my_en_wikipedia_pass")
"""
password_f = open(config.password_file)
for line in password_f:
if not line.strip():
continue
entry = eval(line.decode('utf-8'))
- if len(entry) == 2: # for default userinfo
- if entry[0] == self.username:
- self.password = entry[1]
- elif len(entry) == 4: # for userinfo included code and family
+ if len(entry) == 4: # for userinfo included code and family
if entry[0] == self.site.code and \
entry[1] == self.site.family.name and \
entry[2] == self.username:
self.password = entry[3]
+ elif len(entry) == 3: # for userinfo included family
+ if entry[0] == self.site.family.name and \
+ entry[1] == self.username:
+ self.password = entry[2]
+ elif len(entry) == 2: # for default userinfo
+ if entry[0] == self.username:
+ self.password = entry[1]
password_f.close()
def login(self, retry=False):
--
To view, visit https://gerrit.wikimedia.org/r/127453
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1390e0cc8ff72ee165bf115ef3c9d69e7d2f35af
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio <adriyetichaves(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot <>