jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/314857 )
Change subject: [IMPR] Provide generator settings inside (user-)fixes.py
......................................................................
[IMPR] Provide generator settings inside (user-)fixes.py
- Generator has the keyword 'generator' and may be a str
or a iterable of str which enables multiple pagegenerators arguments
- The fixes generator is invoked only if no generator is given by command
line. This enables to override the default generator given by the fixes set.
Bug: T147418
Change-Id: Id099b50b766fb87678807a2a209c5bd2dd824245
---
M scripts/replace.py
1 file changed, 8 insertions(+), 1 deletion(-)
Approvals:
Huji: Looks good to me, approved
jenkins-bot: Verified
Zoranzoki21: Looks good to me, but someone else must approve
diff --git a/scripts/replace.py b/scripts/replace.py
index 1abd206..cd39f4c 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -128,7 +128,7 @@
"""
#
# (C) Daniel Herding, 2004-2012
-# (C) Pywikibot team, 2004-2016
+# (C) Pywikibot team, 2004-2017
#
# Distributed under the terms of the MIT license.
#
@@ -1023,6 +1023,7 @@
# Perform one of the predefined actions.
missing_fixes_summaries = [] # which a fixes/replacements miss a summary
+ generators_given = bool(genFactory.gens)
for fix_name in fixes_set:
try:
fix = fixes.fixes[fix_name]
@@ -1044,6 +1045,12 @@
set_summary = i18n.translate(site, fix['msg'], fallback=True)
else:
set_summary = None
+ if not generators_given and 'generator' in fix:
+ gen_args = fix['generator']
+ if isinstance(gen_args, basestring):
+ gen_args = [gen_args]
+ for gen_arg in gen_args:
+ genFactory.handleArg(gen_arg)
replacement_set = ReplacementList(fix.get('regex'),
fix.get('exceptions'),
fix.get('nocase'),
--
To view, visit https://gerrit.wikimedia.org/r/314857
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id099b50b766fb87678807a2a209c5bd2dd824245
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Huji <huji.huji(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <dori.gord(a)orion.rs>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/367321 )
Change subject: Raise a new exception class when we get a page with namespace < 0
......................................................................
Raise a new exception class when we get a page with namespace < 0
This could be used to handle this error in scripts more properly
Bug: T169213
Change-Id: Icb312829e575fa85c0c2d477f3e7cb904dd27bfb
---
M pywikibot/__init__.py
M pywikibot/data/api.py
M pywikibot/exceptions.py
3 files changed, 19 insertions(+), 3 deletions(-)
Approvals:
Huji: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py
index e4000cc..348b596 100644
--- a/pywikibot/__init__.py
+++ b/pywikibot/__init__.py
@@ -53,7 +53,7 @@
SiteDefinitionError, NoSuchSite, UnknownSite, UnknownFamily,
UnknownExtension,
NoUsername, UserBlocked,
- PageRelatedError, IsRedirectPage, IsNotRedirectPage,
+ PageRelatedError, UnsupportedPage, IsRedirectPage, IsNotRedirectPage,
PageSaveRelatedError, PageNotSaved, OtherPageSaveError,
LockedPage, CascadeLockedPage, LockedNoPage, NoCreateError,
EditConflict, PageDeletedConflict, PageCreatedConflict,
@@ -106,7 +106,8 @@
'SiteDefinitionError', 'NoSuchSite', 'UnknownSite', 'UnknownFamily',
'UnknownExtension',
'NoUsername', 'UserBlocked', 'UserActionRefuse',
- 'PageRelatedError', 'IsRedirectPage', 'IsNotRedirectPage',
+ 'PageRelatedError', 'UnsupportedPage', 'IsRedirectPage',
+ 'IsNotRedirectPage',
'PageSaveRelatedError', 'PageNotSaved', 'OtherPageSaveError',
'LockedPage', 'CascadeLockedPage', 'LockedNoPage', 'NoCreateError',
'EditConflict', 'PageDeletedConflict', 'PageCreatedConflict',
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 6dc86d6..a84db31 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -35,7 +35,7 @@
from pywikibot.comms import http
from pywikibot.exceptions import (
Server504Error, Server414Error, FatalServerError, NoUsername,
- Error, TimeoutError, InvalidTitle
+ Error, TimeoutError, InvalidTitle, UnsupportedPage
)
from pywikibot.tools import (
MediaWikiVersion, deprecated, itergroup, ip, PY2, getargspec,
@@ -3144,6 +3144,8 @@
property which would make the value present must be in the props
parameter.
@type props: iterable of string
+ @raises InvalidTitle: Page title is invalid
+ @raises UnsupportedPage: Page with namespace < 0 is not supported yet
"""
if "pageid" in pagedict:
page._pageid = int(pagedict['pageid'])
@@ -3154,6 +3156,8 @@
if page.site.sametitle(page.title(), pagedict['title']):
if 'invalid' in pagedict:
raise InvalidTitle('%s: %s' % (page, pagedict['invalidreason']))
+ if int(pagedict['ns']) < 0:
+ raise UnsupportedPage(page)
raise AssertionError(
"Page %s has neither 'pageid' nor 'missing' attribute" % pagedict['title'])
page._contentmodel = pagedict.get('contentmodel') # can be None
diff --git a/pywikibot/exceptions.py b/pywikibot/exceptions.py
index d9dff19..52018df 100644
--- a/pywikibot/exceptions.py
+++ b/pywikibot/exceptions.py
@@ -24,6 +24,7 @@
PageRelatedError: any exception which is caused by an operation on a Page.
- NoPage: Page does not exist
+ - UnsupportedPage: Page is not supported due to a namespace restriction
- IsRedirectPage: Page is a redirect page
- IsNotRedirectPage: Page is not a redirect page
- CircularRedirect: Page is a circular redirect
@@ -227,6 +228,16 @@
pass
+class UnsupportedPage(PageRelatedError):
+
+ """Unsupported page due to namespace restriction."""
+
+ # namespaces < 0 aren't supported (T169213)
+ message = 'Page %s is not supported due to namespace restriction.'
+
+ pass
+
+
class NoMoveTarget(PageRelatedError):
"""Expected move target page not found."""
--
To view, visit https://gerrit.wikimedia.org/r/367321
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icb312829e575fa85c0c2d477f3e7cb904dd27bfb
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dereckson <dereckson(a)espace-win.org>
Gerrit-Reviewer: Huji <huji.huji(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #4327
Status: Broken
Duration: 2 hours, 3 minutes, and 54 seconds
Commit: 681bf23 (master)
Author: dalba
Message: family.py: Simplify category_redirects method
_get_cr_templates always sets a value for _catredirtemplates[code], therefore
checking for `if code in self._catredirtemplates` is not necessary and the
KeyError will never be raised.
Change-Id: Id94bf466063a58bf6311e6a84128f854d09c19d1
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/0309a77e1939...681bf239…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/276726355?utm_source=…
--
You can configure recipients for build notifications in your .travis.yml file. See https://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/378654 )
Change subject: family.py: Simplify category_redirects method
......................................................................
family.py: Simplify category_redirects method
_get_cr_templates always sets a value for _catredirtemplates[code], therefore
checking for `if code in self._catredirtemplates` is not necessary and the
KeyError will never be raised.
Change-Id: Id94bf466063a58bf6311e6a84128f854d09c19d1
---
M pywikibot/family.py
1 file changed, 1 insertion(+), 5 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/family.py b/pywikibot/family.py
index 4f3bd43..b708069 100644
--- a/pywikibot/family.py
+++ b/pywikibot/family.py
@@ -1005,11 +1005,7 @@
if not hasattr(self, "_catredirtemplates") or \
code not in self._catredirtemplates:
self._get_cr_templates(code, fallback)
- if code in self._catredirtemplates:
- return self._catredirtemplates[code]
- else:
- raise KeyError("ERROR: title for category redirect template in "
- "language '%s' unknown" % code)
+ return self._catredirtemplates[code]
def _get_cr_templates(self, code, fallback):
"""Build list of category redirect templates."""
--
To view, visit https://gerrit.wikimedia.org/r/378654
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id94bf466063a58bf6311e6a84128f854d09c19d1
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/373849 )
Change subject: [bugfix] rewrite family._get_cr_templates()
......................................................................
[bugfix] rewrite family._get_cr_templates()
- category_redirect_templates is a tuple, not a list
- retrieve backlinks of all category redirect templates not for the first
item only
- use fallback if fallback is given and key found in template tuple
- keep list order for the given tuple and append the backlinks
Bug: T174041
Change-Id: I596aee7b8934c8aa022cca56679c336c098f5b5b
---
M pywikibot/family.py
1 file changed, 15 insertions(+), 13 deletions(-)
Approvals:
Dalba: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/family.py b/pywikibot/family.py
index 9c94c1b..4f3bd43 100644
--- a/pywikibot/family.py
+++ b/pywikibot/family.py
@@ -1016,23 +1016,25 @@
if not hasattr(self, "_catredirtemplates"):
self._catredirtemplates = {}
if code in self.category_redirect_templates:
- cr_template_list = self.category_redirect_templates[code]
- cr_list = list(self.category_redirect_templates[code])
+ cr_template_tuple = self.category_redirect_templates[code]
+ elif fallback and fallback in self.category_redirect_templates:
+ cr_template_tuple = self.category_redirect_templates[fallback]
else:
- cr_template_list = self.category_redirect_templates[fallback]
- cr_list = []
- if cr_template_list:
- cr_template = cr_template_list[0]
- # start with list of category redirect templates from family file
- cr_page = pywikibot.Page(pywikibot.Site(code, self),
- "Template:" + cr_template)
+ self._catredirtemplates[code] = []
+ return
+ cr_set = set()
+ site = pywikibot.Site(code, self)
+ tpl_ns = site.namespaces.TEMPLATE
+ for cr_template in cr_template_tuple:
+ cr_page = pywikibot.Page(site, cr_template, ns=tpl_ns)
# retrieve all redirects to primary template from API,
# add any that are not already on the list
- for t in cr_page.backlinks(filterRedirects=True, namespaces=10):
+ for t in cr_page.backlinks(filterRedirects=True,
+ namespaces=tpl_ns):
newtitle = t.title(withNamespace=False)
- if newtitle not in cr_list:
- cr_list.append(newtitle)
- self._catredirtemplates[code] = cr_list
+ if newtitle not in cr_template_tuple:
+ cr_set.add(newtitle)
+ self._catredirtemplates[code] = list(cr_template_tuple) + list(cr_set)
@deprecated('site.category_redirects()')
def get_cr_templates(self, code, fallback):
--
To view, visit https://gerrit.wikimedia.org/r/373849
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I596aee7b8934c8aa022cca56679c336c098f5b5b
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Russell Blau <russblau(a)imapmail.org>
Gerrit-Reviewer: Strainu <wiki(a)strainu.ro>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/370664 )
Change subject: [IMPR] Share claimit.py logic with harvest_template.py
......................................................................
[IMPR] Share claimit.py logic with harvest_template.py
It is now possible to import another value for the same property
that the item already has, using -exists:p. Each imported claim
can have its 'exists' setting, as well as all claims can share
the same (global) one.
Additionally, because presence of a property is no longer a reason
to skip adding a value, as well as because of performance, I'm
dropping the safeguard that skips the page when the item already
has all the properties to be harvested.
Bug: T72702
Bug: T76391
Change-Id: I4b4bb9cd2f7427b2226c05212b27d0113163464f
---
M pywikibot/bot.py
M scripts/claimit.py
M scripts/harvest_template.py
3 files changed, 151 insertions(+), 134 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
Ejegg: Looks good to me, but someone else must approve
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 1d95652..3444389 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -2000,6 +2000,69 @@
source.setTarget(item)
return source
+ def user_add_claim_unless_exists(
+ self, item, claim, exists_arg='', source=None,
+ logger_callback=log, **kwargs):
+ """
+ Decorator of L{user_add_claim}.
+
+ Before adding a new claim, it checks if we can add it, using provided
+ filters.
+
+ @see: documentation of L{claimit.py<scripts.claimit>}
+ @param exists_arg: pattern for merging existing claims with new ones
+ @type exists_arg: str
+ @param logger_callback: function logging the output of the method
+ @type logger_callback: callable
+ @return: whether the claim could be added
+ @rtype: bool
+ """
+ # Existing claims on page of same property
+ for existing in item.get().get('claims').get(claim.getID(), []):
+ # If claim with same property already exists...
+ if 'p' not in exists_arg:
+ logger_callback(
+ 'Skipping %s because claim with same property already exists'
+ % (claim.getID(),))
+ log('Use -exists:p option to override this behavior')
+ return False
+ if not existing.target_equals(claim.getTarget()):
+ continue
+ # If some attribute of the claim being added
+ # matches some attribute in an existing claim of
+ # the same property, skip the claim, unless the
+ # 'exists' argument overrides it.
+ if 't' not in exists_arg:
+ logger_callback(
+ 'Skipping %s because claim with same target already exists'
+ % (claim.getID(),))
+ log("Append 't' to -exists argument to override this behavior")
+ return False
+ if 'q' not in exists_arg and not existing.qualifiers:
+ logger_callback(
+ 'Skipping %s because claim without qualifiers already exists'
+ % (claim.getID(),))
+ log("Append 'q' to -exists argument to override this behavior")
+ return False
+ if ('s' not in exists_arg or not source) and not existing.sources:
+ logger_callback(
+ 'Skipping %s because claim without source already exists'
+ % (claim.getID(),))
+ log("Append 's' to -exists argument to override this behavior")
+ return False
+ if ('s' not in exists_arg and source and
+ any(source.getID() in ref and
+ all(snak.target_equals(source.getTarget())
+ for snak in ref[source.getID()])
+ for ref in existing.sources)):
+ logger_callback(
+ 'Skipping %s because claim with the same source already exists'
+ % (claim.getID(),))
+ log("Append 's' to -exists argument to override this behavior")
+ return False
+
+ return self.user_add_claim(item, claim, source, **kwargs)
+
def create_item_for_page(self, page, data=None, summary=None, **kwargs):
"""
Create an ItemPage with the provided page as the sitelink.
diff --git a/scripts/claimit.py b/scripts/claimit.py
index 1f213d0..caf75ec 100755
--- a/scripts/claimit.py
+++ b/scripts/claimit.py
@@ -92,60 +92,10 @@
def treat_page_and_item(self, page, item):
"""Treat each page."""
- # The generator might yield pages from multiple sites
- source = self.getSource(page.site)
-
for claim in self.claims:
- # Existing claims on page of same property
- for existing in item.claims.get(claim.getID(), []):
- # If claim with same property already exists...
- if 'p' not in self.exists_arg:
- pywikibot.log(
- 'Skipping %s because claim with same property already exists'
- % (claim.getID(),))
- pywikibot.log(
- 'Use -exists:p option to override this behavior')
- break
- if not existing.target_equals(claim.getTarget()):
- continue
- # If some attribute of the claim being added
- # matches some attribute in an existing claim of
- # the same property, skip the claim, unless the
- # 'exists' argument overrides it.
- if 't' not in self.exists_arg:
- pywikibot.log(
- 'Skipping %s because claim with same target already exists'
- % (claim.getID(),))
- pywikibot.log(
- "Append 't' to -exists argument to override this behavior")
- break
- if 'q' not in self.exists_arg and not existing.qualifiers:
- pywikibot.log(
- 'Skipping %s because claim without qualifiers already exists'
- % (claim.getID(),))
- pywikibot.log(
- "Append 'q' to -exists argument to override this behavior")
- break
- if ('s' not in self.exists_arg or not source) and not existing.sources:
- pywikibot.log(
- 'Skipping %s because claim without source already exists'
- % (claim.getID(),))
- pywikibot.log(
- "Append 's' to -exists argument to override this behavior")
- break
- if ('s' not in self.exists_arg and source and
- any(source.getID() in ref and
- all(snak.target_equals(source.getTarget())
- for snak in ref[source.getID()])
- for ref in existing.sources)):
- pywikibot.log(
- 'Skipping %s because claim with the same source already exists'
- % (claim.getID(),))
- pywikibot.log(
- "Append 's' to -exists argument to override this behavior")
- break
- else:
- self.user_add_claim(item, claim, page.site)
+ # The generator might yield pages from multiple sites
+ self.user_add_claim_unless_exists(
+ item, claim, self.exists_arg, page.site)
def main(*args):
diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py
index 7303524..9d76d23 100755
--- a/scripts/harvest_template.py
+++ b/scripts/harvest_template.py
@@ -34,6 +34,12 @@
-islink Treat plain text values as links ("text" -> "[[text]]").
+-exists If set to 'p', add a new value, even if the item already
+ has the imported property but not the imported value.
+ If set to 'pt', add a new value, even if the item already
+ has the imported property with the imported value and
+ some qualifiers.
+
Examples:
python pwb.py harvest_template -lang:en -family:wikipedia -namespace:0 \
@@ -60,6 +66,14 @@
-template:"Infobox person" birth_place P19 -islink death_place P20
will do the same but only "birth_place" can be imported without a link.
+
+ python pwb.py harvest_template -lang:en -family:wikipedia -namespace:0 \
+ -template:"Infobox person" occupation P106 -exists:p
+
+ will import an occupation from "occupation" parameter of "Infobox
+ person" on English Wikipedia as Wikidata property "P106" (occupation). The
+ page won't be skipped if the item already has that property but there is
+ not the new value.
"""
#
@@ -100,6 +114,7 @@
availableOptions = {
'islink': False,
+ 'exists': '',
}
@@ -121,10 +136,14 @@
@type islink: bool
@keyword create: Whether to create a new item if it's missing
@type create: bool
+ @keyword exists: pattern for merging existing claims with harvested
+ values
+ @type exists: str
"""
self.availableOptions.update({
'always': True,
'create': False,
+ 'exists': '',
'islink': False,
})
super(HarvestRobot, self).__init__(**kwargs)
@@ -197,25 +216,19 @@
"""
Compare bot's (global) and provided (local) options.
- @see: L{pywikibot.bot.OptionHandler.getOption}
-
- @rtype: bool
+ @see: L{OptionHandler.getOption}
"""
- # TODO: only works with booleans
default = self.getOption(option)
local = handler.getOption(option)
- return default is not local
+ if isinstance(default, bool) and isinstance(local, bool):
+ return default is not local
+ else:
+ return local or default
def treat_page_and_item(self, page, item):
"""Process a single page/item."""
if willstop:
raise KeyboardInterrupt
- item.get()
- if set(val[0] for val in self.fields.values()) <= set(
- item.claims.keys()):
- pywikibot.output('%s item %s has claims for all properties. '
- 'Skipping.' % (page, item.title()))
- return
templates = page.raw_extracted_templates
for (template, fielddict) in templates:
@@ -228,80 +241,71 @@
"Failed parsing template; '%s' should be the template name."
% template)
continue
+
+ if template not in self.templateTitles:
+ continue
# We found the template we were looking for
- if template in self.templateTitles:
- for field, value in fielddict.items():
- field = field.strip()
- value = value.strip()
- if not field or not value:
+ for field, value in fielddict.items():
+ field = field.strip()
+ value = value.strip()
+ if not field or not value:
+ continue
+
+ if field not in self.fields:
+ continue
+
+ # This field contains something useful for us
+ prop, options = self.fields[field]
+ claim = pywikibot.Claim(self.repo, prop)
+ if claim.type == 'wikibase-item':
+ # Try to extract a valid page
+ match = pywikibot.link_regex.search(value)
+ if match:
+ link_text = match.group(1)
+ else:
+ if self._get_option_with_fallback(options, 'islink'):
+ link_text = value
+ else:
+ pywikibot.output(
+ '%s field %s value %s is not a wikilink. '
+ 'Skipping.' % (claim.getID(), field, value))
+ continue
+
+ linked_item = self._template_link_target(item, link_text)
+ if not linked_item:
continue
- # This field contains something useful for us
- if field in self.fields:
- prop, options = self.fields[field]
- # Check if the property isn't already set
- claim = pywikibot.Claim(self.repo, prop)
- if claim.getID() in item.get().get('claims'):
- pywikibot.output(
- 'A claim for %s already exists. Skipping.'
- % claim.getID())
- # TODO: Implement smarter approach to merging
- # harvested values with existing claims esp.
- # without overwriting humans unintentionally.
- else:
- if claim.type == 'wikibase-item':
- # Try to extract a valid page
- match = pywikibot.link_regex.search(value)
- if match:
- link_text = match.group(1)
- else:
- if self._get_option_with_fallback(
- options, 'islink'):
- link_text = value
- else:
- pywikibot.output(
- '%s field %s value %s is not a '
- 'wikilink. Skipping.'
- % (claim.getID(), field, value))
- continue
+ claim.setTarget(linked_item)
+ elif claim.type in ('string', 'external-id'):
+ claim.setTarget(value.strip())
+ elif claim.type == 'url':
+ match = self.linkR.search(value)
+ if not match:
+ continue
+ claim.setTarget(match.group('url'))
+ elif claim.type == 'commonsMedia':
+ commonssite = pywikibot.Site('commons', 'commons')
+ imagelink = pywikibot.Link(
+ value, source=commonssite, defaultNamespace=6)
+ image = pywikibot.FilePage(imagelink)
+ if image.isRedirectPage():
+ image = pywikibot.FilePage(image.getRedirectTarget())
+ if not image.exists():
+ pywikibot.output(
+ "{0} doesn't exist. I can't link to it"
+ ''.format(image.title(asLink=True)))
+ continue
+ claim.setTarget(image)
+ else:
+ pywikibot.output('%s is not a supported datatype.'
+ % claim.type)
+ continue
- linked_item = self._template_link_target(
- item, link_text)
- if not linked_item:
- continue
-
- claim.setTarget(linked_item)
- elif claim.type in ('string', 'external-id'):
- claim.setTarget(value.strip())
- elif claim.type == 'url':
- match = self.linkR.search(value)
- if not match:
- continue
- claim.setTarget(match.group('url'))
- elif claim.type == 'commonsMedia':
- commonssite = pywikibot.Site('commons',
- 'commons')
- imagelink = pywikibot.Link(value,
- source=commonssite,
- defaultNamespace=6)
- image = pywikibot.FilePage(imagelink)
- if image.isRedirectPage():
- image = pywikibot.FilePage(
- image.getRedirectTarget())
- if not image.exists():
- pywikibot.output(
- "{0} doesn't exist. I can't link to it"
- ''.format(image.title(asLink=True)))
- continue
- claim.setTarget(image)
- else:
- pywikibot.output(
- '%s is not a supported datatype.'
- % claim.type)
- continue
-
- # A generator might yield pages from multiple sites
- self.user_add_claim(item, claim, page.site)
+ # A generator might yield pages from multiple sites
+ self.user_add_claim_unless_exists(
+ item, claim, self._get_option_with_fallback(
+ options, 'exists'),
+ page.site, pywikibot.output)
def main(*args):
--
To view, visit https://gerrit.wikimedia.org/r/370664
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4b4bb9cd2f7427b2226c05212b27d0113163464f
Gerrit-PatchSet: 10
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Ejegg <ejegg(a)ejegg.com>
Gerrit-Reviewer: JAn Dudík <jan.dudik(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <Ladsgroup(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XXN <dan10real(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/378377 )
Change subject: Fix test_unconnected and test_unconnected_with_repo
......................................................................
Fix test_unconnected and test_unconnected_with_repo
site_tests.test_unconnected:
Sometimes local site is not synchronized with data repository. In
those situations unconnected_pages may return a page that is actually
connected to wikidata and test_unconnected may fail. (T175987)
To avoid this situation, instead of confirming that unconnected pages have
no item on wikidata, only confirm that the right query is sent, i.e. with
right parameters and via UnconnectedPages.
It's Mediawiki's job to ensure the correct results when a correct request
is sent.
pagegenerators_tests.test_unconnected_with_repo:
Instead of repeating the logic of site_tests.test_unconnected, just
assert that unconnected_pages is called with right parameters.
Now that these tests are not time-dependant anymore we can cache the results.
Add `cached = True` to TestUnconnectedPageGenerator and put test_unconnected
in TestSiteGenerators class (which is cached).
Also no need to check for `self.site.hostname() == 'test.wikipedia.org'`
condition because we are not relying on the result of the related data
repositoy in these tests.
Remove unused imports.
Bug: T163619
Change-Id: I625401eb7f5ad02852f5652df706d2264dd6a6f9
---
M tests/pagegenerators_tests.py
M tests/site_tests.py
2 files changed, 30 insertions(+), 57 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 20d1deb..b88770a 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -9,11 +9,15 @@
import calendar
import datetime
-import json
import logging
import sys
from distutils.version import LooseVersion
+
+try:
+ from unittest.mock import patch
+except ImportError:
+ from mock import patch
import pywikibot
from pywikibot import pagegenerators, date
@@ -1455,34 +1459,21 @@
"""Test UnconnectedPageGenerator."""
+ cached = True
+
def test_unconnected_with_repo(self):
- """Test that the ItemPage returned raises NoPage."""
+ """Test UnconnectedPageGenerator."""
if not self.site.data_repository():
raise unittest.SkipTest('Site is not using a Wikibase repository')
- if self.site.hostname() == 'test.wikipedia.org':
- raise unittest.SkipTest('test.wikipedia is misconfigured; T85358')
- cnt = 0
- # Pages which have been connected recently may still be reported as
- # unconnected. So try on a version that is a few minutes older if the
- # tested site appears as a sitelink.
- start_time = \
- datetime.datetime.utcnow() - datetime.timedelta(minutes=30)
- for page in pagegenerators.UnconnectedPageGenerator(self.site, 3):
- try:
- item = pywikibot.ItemPage.fromPage(page)
- except pywikibot.NoPage:
- pass
- else:
- revisions = list(item.revisions(total=1, starttime=start_time,
- content=True))
- if revisions:
- sitelinks = json.loads(revisions[0].text)['sitelinks']
- self.assertNotIn(
- self.site.dbName(), sitelinks,
- 'Page "{0}" is connected to {1} on Wikibase '
- 'repository'.format(page.title(), item))
- cnt += 1
- self.assertLessEqual(cnt, 3)
+ upgen = pagegenerators.UnconnectedPageGenerator(self.site, 3)
+
+ def unconnected_pages(total=None):
+ """Assert unconnected_pages is called correctly."""
+ self.assertEqual(total, 3)
+ yield
+
+ with patch.object(self.site, 'unconnected_pages', unconnected_pages):
+ self.assertEqual(tuple(upgen), (None,))
def test_unconnected_without_repo(self):
"""Test that it raises a ValueError on sites without repository."""
diff --git a/tests/site_tests.py b/tests/site_tests.py
index add575c..996389d 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -7,14 +7,13 @@
#
from __future__ import absolute_import, unicode_literals
-import json
import os
import pickle
import re
import sys
from collections import Iterable, Mapping
-from datetime import datetime, timedelta
+from datetime import datetime
import pywikibot
@@ -1034,38 +1033,21 @@
self.fail(
'NotImplementedError not raised for {0}'.format(item))
-
-class TestSiteGeneratorsUncached(DefaultSiteTestCase):
-
- """Test cases for Site methods."""
-
def test_unconnected(self):
- """Test that the ItemPage returned raises NoPage."""
+ """Test site.unconnected_pages method."""
if not self.site.data_repository():
raise unittest.SkipTest('Site is not using a Wikibase repository')
- if self.site.hostname() == 'test.wikipedia.org':
- raise unittest.SkipTest('test.wikipedia is misconfigured; T85358')
- cnt = 0
- # Pages which have been connected recently may still be reported as
- # unconnected. So try on a version that is a few minutes older if the
- # tested site appears as a sitelink.
- start_time = datetime.utcnow() - timedelta(minutes=30)
- for page in self.site.unconnected_pages(total=3):
- try:
- item = pywikibot.ItemPage.fromPage(page)
- except pywikibot.NoPage:
- pass
- else:
- revisions = list(item.revisions(total=1, starttime=start_time,
- content=True))
- if revisions:
- sitelinks = json.loads(revisions[0].text)['sitelinks']
- self.assertNotIn(
- self.site.dbName(), sitelinks,
- 'Page "{0}" is connected to {1} on Wikibase '
- 'repository'.format(page.title(), item))
- cnt += 1
- self.assertLessEqual(cnt, 3)
+ upgen = self.site.unconnected_pages(total=3)
+ self.assertDictEqual(
+ upgen.request._params, {
+ 'gqppage': ['UnconnectedPages'],
+ 'prop': ['info', 'imageinfo', 'categoryinfo'],
+ 'inprop': ['protection'],
+ 'iiprop': ['timestamp', 'user', 'comment', 'url', 'size',
+ 'sha1', 'metadata'],
+ 'generator': ['querypage'], 'action': ['query'],
+ 'indexpageids': [True], 'continue': [True]})
+ self.assertLessEqual(len(tuple(upgen)), 3)
class TestImageUsage(DefaultSiteTestCase):
--
To view, visit https://gerrit.wikimedia.org/r/378377
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I625401eb7f5ad02852f5652df706d2264dd6a6f9
Gerrit-PatchSet: 6
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/378380 )
Change subject: _http_param_string.__doc__: Add a note about parameters not being sorted
......................................................................
_http_param_string.__doc__: Add a note about parameters not being sorted
Change-Id: I5e2295c3a9d0e6150dd5640f18e2101867d0aed1
---
M pywikibot/data/api.py
1 file changed, 4 insertions(+), 0 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index a88d9e4..b9ff3ae 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -1771,6 +1771,10 @@
Return the parameters as a HTTP URL query fragment.
URL encodes the parameters provided by _encoded_items()
+
+ @note: Not all parameters are sorted, therefore for two given
+ CachedRequest objects with equal _params, the result of
+ _http_param_string() is not necessarily equal.
"""
return encode_url(self._encoded_items())
--
To view, visit https://gerrit.wikimedia.org/r/378380
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5e2295c3a9d0e6150dd5640f18e2101867d0aed1
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/377677 )
Change subject: tests.aspects.TestCase.has_site_user: Support '*' as family name
......................................................................
tests.aspects.TestCase.has_site_user: Support '*' as family name
Pywikibot supports '*' as family name since a10cbb30ed58085c754519e066c72d.
Tests that require a username should support this new method of specifying
usernames in userconfig.py and should not be skipped.
Bug: T120334
Change-Id: I278d70127bfc4f8d26035b74c3ae88dda62e37de
---
M tests/aspects.py
1 file changed, 2 insertions(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/tests/aspects.py b/tests/aspects.py
index 76fd2b5..9a60cfc 100644
--- a/tests/aspects.py
+++ b/tests/aspects.py
@@ -1018,7 +1018,8 @@
usernames = config.sysopnames if sysop else config.usernames
- return code in usernames[family] or '*' in usernames[family]
+ return (code in usernames[family] or '*' in usernames[family] or
+ code in usernames['*'] or '*' in usernames['*'])
def __init__(self, *args, **kwargs):
"""Constructor."""
--
To view, visit https://gerrit.wikimedia.org/r/377677
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I278d70127bfc4f8d26035b74c3ae88dda62e37de
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/378066 )
Change subject: Fix authentication of flickrripper.py
......................................................................
Fix authentication of flickrripper.py
No need to get a token anymore. The underlying library handles
all the oauth stuff.
Bug: T173348
Change-Id: Ic177c3c9c960ced22f059436bdc876770fd4e65a
---
M scripts/flickrripper.py
1 file changed, 0 insertions(+), 6 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py
index 7fd1fb1..8afd672 100755
--- a/scripts/flickrripper.py
+++ b/scripts/flickrripper.py
@@ -431,12 +431,6 @@
if 'api_secret' in config.flickr and config.flickr['api_secret']:
flickr = flickrapi.FlickrAPI(config.flickr['api_key'], config.flickr['api_secret'])
- (token, frob) = flickr.get_token_part_one(perms='read')
- if not token:
- # The user still hasn't authorised this app yet, get_token_part_one()
- # will have spawn a browser window
- pywikibot.input("Press ENTER after you authorized this program")
- flickr.get_token_part_two((token, frob))
else:
pywikibot.output('Accessing public content only')
flickr = flickrapi.FlickrAPI(config.flickr['api_key'])
--
To view, visit https://gerrit.wikimedia.org/r/378066
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic177c3c9c960ced22f059436bdc876770fd4e65a
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Multichill <maarten(a)mdammers.nl>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/378201 )
Change subject: site_tests.test_search_where_title: Use assertIn instead of assertTrue
......................................................................
site_tests.test_search_where_title: Use assertIn instead of assertTrue
This way unittest will produce a more descriptive error message by showing
AssertionError: 'wiki' not found in <page title>
instead of
AssertionError: False is not true
Change-Id: I98059d53d5b2be1673743503e4809ceab7e7e869
---
M tests/site_tests.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/tests/site_tests.py b/tests/site_tests.py
index 74d3b11..add575c 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -1505,7 +1505,7 @@
get_redirects=True, where='title'):
self.assertIsInstance(hit, pywikibot.Page)
self.assertEqual(hit.namespace(), 0)
- self.assertTrue('wiki' in hit.title().lower())
+ self.assertIn('wiki', hit.title().lower())
except pywikibot.data.api.APIError as e:
if e.code in ('search-title-disabled', 'gsrsearch-title-disabled'):
raise unittest.SkipTest(
--
To view, visit https://gerrit.wikimedia.org/r/378201
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I98059d53d5b2be1673743503e4809ceab7e7e869
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #4313
Status: Passed
Duration: 2 hours, 8 minutes, and 51 seconds
Commit: 22d19cf (master)
Author: dalba
Message: dev-requirements.txt: Avoide double-requirement
Mentioning the inverse python_version of line 11 in line 12 prevents
double-requirement situation which may occure under Python 2.6 and Windows OS.
Bug: T175669
Change-Id: Iadf7191b7794ea4a0863424a36d138d2fe2dbd16
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/e121eb8e6027...22d19cf4…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/274803957?utm_source=…
--
You can configure recipients for build notifications in your .travis.yml file. See https://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/376058 )
Change subject: [IMPR] Cache fetched Wikibase item
......................................................................
[IMPR] Cache fetched Wikibase item
Repetitive ItemPage.fromPage calls always create
a new instance. When using -onlyif(not) filters,
the item's content can be fetched during filtering
and during treating.
Change-Id: I5b5b4d69da32c311f7d5ffe7d1767bc31e73b295
---
M pywikibot/page.py
1 file changed, 8 insertions(+), 3 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/page.py b/pywikibot/page.py
index c03c878..d54be98 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -163,7 +163,7 @@
'_contentmodel', '_langlinks', '_isredir', '_coords',
'_preloadedtext', '_timestamp', '_applicable_protections',
'_flowinfo', '_quality', '_pageprops', '_revid', '_quality_text',
- '_pageimage'
+ '_pageimage', '_item'
)
def __init__(self, source, title=u"", ns=0):
@@ -4178,6 +4178,8 @@
@raise NoPage: There is no corresponding ItemPage for the page
@raise WikiBaseError: The site of the page has no data repository.
"""
+ if hasattr(page, '_item'):
+ return page._item
if not page.site.has_data_repository:
raise pywikibot.WikiBaseError('{0} has no data repository'
''.format(page.site))
@@ -4189,7 +4191,8 @@
'_pageprops') and page.properties().get('wikibase_item'):
# If we have already fetched the pageprops for something else,
# we already have the id, so use it
- return cls(repo, page.properties().get('wikibase_item'))
+ page._item = cls(repo, page.properties().get('wikibase_item'))
+ return page._item
i = cls(repo)
# clear id, and temporarily store data needed to lazy loading the item
del i.id
@@ -4197,7 +4200,8 @@
i._title = page.title(withSection=False)
if not lazy_load and not i.exists():
raise pywikibot.NoPage(i)
- return i
+ page._item = i
+ return page._item
@classmethod
def from_entity_uri(cls, site, uri, lazy_load=False):
@@ -4309,6 +4313,7 @@
for dbname in self.sitelinks:
pg = Page(pywikibot.site.APISite.fromDBName(dbname),
self.sitelinks[dbname])
+ pg._item = self
if family is None or family == pg.site.family:
yield pg
--
To view, visit https://gerrit.wikimedia.org/r/376058
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5b5b4d69da32c311f7d5ffe7d1767bc31e73b295
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375833 )
Change subject: Only preload grepping generator if possible
......................................................................
Only preload grepping generator if possible
When nopreload is True, preloading should always be avoided.
It isn't problem to move this filter after preloading since
it always needs the pages to have content in order to decide.
Change-Id: I9e4c30756660542c481521f985245e5d32ad02f5
---
M pywikibot/pagegenerators.py
1 file changed, 5 insertions(+), 5 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 1216f84..b0c6dab 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -528,20 +528,20 @@
dupfiltergen = RegexFilterPageGenerator(
dupfiltergen, self.titlenotfilter_list, 'none')
- if self.articlefilter_list:
- dupfiltergen = RegexBodyFilterPageGenerator(
- PreloadingGenerator(dupfiltergen), self.articlefilter_list)
-
if self.catfilter_list:
dupfiltergen = CategoryFilterPageGenerator(
dupfiltergen, self.catfilter_list, self.site)
- if preload and not self.nopreload:
+ if (preload or self.articlefilter_list) and not self.nopreload:
if isinstance(dupfiltergen, DequeGenerator):
dupfiltergen = DequePreloadingGenerator(dupfiltergen)
else:
dupfiltergen = PreloadingGenerator(dupfiltergen)
+ if self.articlefilter_list:
+ dupfiltergen = RegexBodyFilterPageGenerator(
+ dupfiltergen, self.articlefilter_list)
+
return dupfiltergen
@deprecated_args(arg='category')
--
To view, visit https://gerrit.wikimedia.org/r/375833
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9e4c30756660542c481521f985245e5d32ad02f5
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/376701 )
Change subject: travis.yml: Disable OAuth jobs
......................................................................
travis.yml: Disable OAuth jobs
It's been nearly 3 weeks since these jobs started to fail.
Each of them takes nearly 50 minutes to complete and at the end there is no
useful output. They are just making the builds slow, distracting us from
things that really need attention and can be fixed.
I also sent a message to the mailing list which has had no reply so far.[1]
[1]: https://lists.wikimedia.org/pipermail/pywikibot/2017-September/009775.html
Bug: T173498
Change-Id: I45f7c4048cea4eb7a135b1c6ea1d582d43afbae7
---
M .travis.yml
1 file changed, 8 insertions(+), 6 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/.travis.yml b/.travis.yml
index c5a9928..554cdfe 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -153,18 +153,20 @@
- graphviz
- liblua5.1-0-dev
- python-ipaddr
- - python: '2.7'
- env: LANGUAGE=en FAMILY=wpbeta SITE_ONLY=1 OAUTH_DOMAIN="en.wikipedia.beta.wmflabs.org"
- - python: '3.3'
- env: LANGUAGE=zh FAMILY=wpbeta SITE_ONLY=1 OAUTH_DOMAIN="zh.wikipedia.beta.wmflabs.org"
+ # Disabled due to T173498
+ #- python: '2.7'
+ # env: LANGUAGE=en FAMILY=wpbeta SITE_ONLY=1 OAUTH_DOMAIN="en.wikipedia.beta.wmflabs.org"
+ #- python: '3.3'
+ # env: LANGUAGE=zh FAMILY=wpbeta SITE_ONLY=1 OAUTH_DOMAIN="zh.wikipedia.beta.wmflabs.org"
- python: '3.4'
env: LANGUAGE=en FAMILY=wsbeta SITE_ONLY=1
- python: '2.7'
env: LANGUAGE=wikia FAMILY=wikia PYWIKIBOT2_TEST_NO_RC=1
- python: '3.3'
env: LANGUAGE=en FAMILY=musicbrainz SITE_ONLY=1
- - python: '3.4'
- env: LANGUAGE=test FAMILY=wikipedia SITE_ONLY=1 OAUTH_DOMAIN="test.wikipedia.org"
+ # Disabled due to T173498
+ #- python: '3.4'
+ # env: LANGUAGE=test FAMILY=wikipedia SITE_ONLY=1 OAUTH_DOMAIN="test.wikipedia.org"
- python: '3.4'
env: LANGUAGE=test FAMILY=wikidata SITE_ONLY=1
- python: '3.4'
--
To view, visit https://gerrit.wikimedia.org/r/376701
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I45f7c4048cea4eb7a135b1c6ea1d582d43afbae7
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/376567 )
Change subject: [Doc] Clean up uses of '¶ms;' placeholder
......................................................................
[Doc] Clean up uses of '¶ms;' placeholder
Change-Id: I1fa0410e67c52c22d2391b97155730f45ec64498
---
M scripts/claimit.py
M scripts/coordinate_import.py
M scripts/match_images.py
M scripts/patrol.py
M scripts/revertbot.py
M scripts/script_wui.py
M scripts/surnames_redirects.py
7 files changed, 12 insertions(+), 10 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/scripts/claimit.py b/scripts/claimit.py
index 845ad49..5741732 100755
--- a/scripts/claimit.py
+++ b/scripts/claimit.py
@@ -3,6 +3,10 @@
"""
A script that adds claims to Wikidata items based on a list of pages.
+These command line parameters can be used to specify which pages to work on:
+
+¶ms;
+
------------------------------------------------------------------------------
Usage:
diff --git a/scripts/coordinate_import.py b/scripts/coordinate_import.py
index 41f6b47..1a1f3e0 100755
--- a/scripts/coordinate_import.py
+++ b/scripts/coordinate_import.py
@@ -21,6 +21,8 @@
python pwb.py coordinate_import -lang:it -family:wikipedia \
-namespace:0 -transcludes:Infobox_stazione_ferroviaria
+These command line parameters can be used to specify which pages to work on:
+
¶ms;
"""
#
@@ -35,6 +37,8 @@
from pywikibot import pagegenerators, WikidataBot
from pywikibot.exceptions import CoordinateGlobeUnknownException
+docuReplacements = {'¶ms;': pagegenerators.parameterHelp}
+
class CoordImportRobot(WikidataBot):
diff --git a/scripts/match_images.py b/scripts/match_images.py
index d618bca..b5c8923 100755
--- a/scripts/match_images.py
+++ b/scripts/match_images.py
@@ -9,8 +9,6 @@
It is essential to provide two images to work on.
-¶ms;
-
Furthermore, the following command line parameters are supported:
-otherfamily Mentioned family with this parameter will be preferred for
diff --git a/scripts/patrol.py b/scripts/patrol.py
index bc3bdc9..98de168 100755
--- a/scripts/patrol.py
+++ b/scripts/patrol.py
@@ -27,6 +27,8 @@
https://en.wikisource.org/wiki/User:Wikisource-bot/patrol_whitelist
+¶ms;
+
Commandline parameters
======================
diff --git a/scripts/revertbot.py b/scripts/revertbot.py
index d417347..89d8d4b 100755
--- a/scripts/revertbot.py
+++ b/scripts/revertbot.py
@@ -5,8 +5,6 @@
The following command line parameters are supported:
-¶ms;
-
-username Edits of which user need to be reverted.
-rollback Rollback edits instead of reverting them.
diff --git a/scripts/script_wui.py b/scripts/script_wui.py
index e552a4e..839bf24 100755
--- a/scripts/script_wui.py
+++ b/scripts/script_wui.py
@@ -11,12 +11,6 @@
(some code might get compiled on-the-fly, so a GNU compiler along
with library header files is needed too)
-The following parameters are supported:
-
-¶ms;
-
-All other parameters will be ignored.
-
Syntax example:
python pwb.py script_wui -dir:.
diff --git a/scripts/surnames_redirects.py b/scripts/surnames_redirects.py
index d817fda..ab19916 100755
--- a/scripts/surnames_redirects.py
+++ b/scripts/surnames_redirects.py
@@ -28,6 +28,8 @@
from pywikibot import i18n, pagegenerators
from pywikibot.bot import FollowRedirectPageBot, ExistingPageBot
+docuReplacements = {'¶ms;': pagegenerators.parameterHelp}
+
class SurnamesBot(ExistingPageBot, FollowRedirectPageBot):
--
To view, visit https://gerrit.wikimedia.org/r/376567
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1fa0410e67c52c22d2391b97155730f45ec64498
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #4296
Status: Errored
Duration: 4 hours, 23 minutes, and 42 seconds
Commit: 6374cae (master)
Author: dalba
Message: Fix regressions of api.QueryGenerator caused by 64da531a33d13b9ca
Presence of `QueryGenerator._check_result_namespace` is not enough for
calling it; `self._namespaces` must be available, too. Use `_namespaces`
as the condition instead.
Change-Id: Ie0d9bd027222786f969f7fa7660459b5fb810ea5
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/5b1fecbdef0f...6374caec…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/272347216?utm_source=…
--
You can configure recipients for build notifications in your .travis.yml file. See https://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/376061 )
Change subject: Fix regressions of api.QueryGenerator caused by 64da531a33d13b9ca
......................................................................
Fix regressions of api.QueryGenerator caused by 64da531a33d13b9ca
Presence of `QueryGenerator._check_result_namespace` is not enough for
calling it; `self._namespaces` must be available, too. Use `_namespaces`
as the condition instead.
Change-Id: Ie0d9bd027222786f969f7fa7660459b5fb810ea5
---
M pywikibot/data/api.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index c057b67..a88d9e4 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -2816,7 +2816,7 @@
self.normalized = {}
for item in resultdata:
result = self.result(item)
- if self._check_result_namespace is not NotImplemented:
+ if self._namespaces:
if not self._check_result_namespace(result):
continue
yield result
--
To view, visit https://gerrit.wikimedia.org/r/376061
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie0d9bd027222786f969f7fa7660459b5fb810ea5
Gerrit-PatchSet: 6
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/366206 )
Change subject: Handle non-items in ItemClaimFilterPageGenerator
......................................................................
Handle non-items in ItemClaimFilterPageGenerator
Now if there are non-items in the generator, we try
to find the item first and then filter on it. However,
we have to abandon preloading since we can no longer
assert that we are working on a list of items.
I have also made the type check weaker, so that other
types of entities are supported as well.
Bug: T137842
Bug: T160396
Bug: T170991
Change-Id: I06851f18d85228d5db77cdec5ed1e07fcda32f7a
---
M pywikibot/pagegenerators.py
1 file changed, 5 insertions(+), 4 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index a38d7d2..bed56fd 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -511,7 +511,6 @@
dupfiltergen, self.subpage_max_depth)
if self.claimfilter_list:
- dupfiltergen = PreloadingItemGenerator(dupfiltergen)
for claim in self.claimfilter_list:
dupfiltergen = ItemClaimFilterPageGenerator(dupfiltergen,
claim[0], claim[1],
@@ -1577,9 +1576,11 @@
@return: true if page contains the claim, false otherwise
@rtype: bool
"""
- if not isinstance(page, pywikibot.ItemPage):
- pywikibot.output(u'%s is not an ItemPage. Skipping.' % page)
- return False
+ if not isinstance(page, pywikibot.WikibasePage):
+ try:
+ page = pywikibot.ItemPage.fromPage(page)
+ except pywikibot.NoPage:
+ return False
for page_claim in page.get()['claims'].get(prop, []):
if page_claim.target_equals(claim):
if not qualifiers:
--
To view, visit https://gerrit.wikimedia.org/r/366206
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I06851f18d85228d5db77cdec5ed1e07fcda32f7a
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: JAn Dudík <jan.dudik(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375719 )
Change subject: QueryGenerator: Allow filtering namespaces after API fetch
......................................................................
QueryGenerator: Allow filtering namespaces after API fetch
- Provide a way to bypass the TypeError of QueryGenerator.set_namespace when
multiple namespaces are given but module does not support multiple
namespaces.
- Modify QueryGenerator.__iter__ slightly to allow filtering the results after
they have been fetched from the API.
- Revert 5cf7fb4275847ab39cafd0e974a3c0cba9c70213. It is not neccessary
anymore. Both `logevents` and `logpages` now support multiple namespaces.
- Add a test for the new functionality of LogEntryListGenerator.
Bug: T174899
Change-Id: Ic523845fd2b143be328ee6edc3b3c53783486026
---
M pywikibot/data/api.py
M pywikibot/site.py
M tests/site_tests.py
3 files changed, 38 insertions(+), 7 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 6dc86d6..c057b67 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -2507,6 +2507,16 @@
"""
+ # Should results be filtered during iteration according to set_namespace?
+ # Used if the API module does not support multiple namespaces.
+ # Override in subclasses by defining a function that returns True if
+ # the result's namespace is in self._namespaces.
+ _check_result_namespace = NotImplemented
+
+ # Set of allowed namespaces will be assigned to _namespaces during
+ # set_namespace call. Only to be used by _check_result_namespace.
+ _namespaces = None
+
def __init__(self, **kwargs):
"""
Construct a QueryGenerator object.
@@ -2683,8 +2693,12 @@
self.site.namespaces.resolve(namespaces)]
if 'multi' not in param and len(namespaces) != 1:
- raise TypeError(u'{0} module does not support multiple namespaces'
- .format(self.limited_module))
+ if self._check_result_namespace is NotImplemented:
+ raise TypeError('{0} module does not support multiple '
+ 'namespaces'.format(self.limited_module))
+ else:
+ self._namespaces = set(namespaces)
+ namespaces = None
if namespaces:
self.request[self.prefix + 'namespace'] = namespaces
@@ -2801,7 +2815,11 @@
else:
self.normalized = {}
for item in resultdata:
- yield self.result(item)
+ result = self.result(item)
+ if self._check_result_namespace is not NotImplemented:
+ if not self._check_result_namespace(result):
+ continue
+ yield result
if isinstance(item, dict) and set(self.continuekey) & set(item.keys()):
# if we need to count elements contained in items in
# self.data["query"]["pages"], we want to count
@@ -3015,6 +3033,10 @@
"""Instatiate LogEntry from data from api."""
return self.entryFactory.create(pagedata)
+ def _check_result_namespace(self, result):
+ """Return True if result.ns() is in self._namespaces."""
+ return result.ns() in self._namespaces
+
class LoginManager(login.LoginManager):
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 2b81e4d..f637342 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -4509,8 +4509,11 @@
@type user: basestring
@param page: only iterate entries affecting this page
@type page: Page or basestring
- @param namespace: namespace to retrieve logevents from
- @type namespace: int or Namespace
+ @param namespace: namespace(s) to retrieve logevents from
+ @type namespace: int or Namespace or an iterable of them
+ @note: due to an API limitation, if namespace param contains multiple
+ namespaces, log entries from all namespaces will be fetched from
+ the API and will be filtered later during iteration.
@param start: only iterate entries from and after this Timestamp
@type start: Timestamp or ISO date string
@param end: only iterate entries up to and through this Timestamp
@@ -4544,8 +4547,8 @@
legen.request["leend"] = end
if reverse:
legen.request["ledir"] = "newer"
- if namespace or namespace == 0:
- legen.request["lenamespace"] = namespace
+ if namespace is not None:
+ legen.set_namespace(namespace)
if tag:
# Supported in version 1.16+; earlier sites will cause APIError
legen.request['letag'] = tag
diff --git a/tests/site_tests.py b/tests/site_tests.py
index 2740a44..1a7f2b8 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -1234,6 +1234,12 @@
entry[2], long if PY2 and entry[2] > sys.maxint else int)
self.assertIsInstance(entry[3], basestring)
+ def test_list_namespace(self):
+ """Test the deprecated site.logpages() when namespace is a list."""
+ le = list(self.site.logpages(namespace=[2, 3], number=10))
+ for entry in le:
+ self.assertIn(entry[0].namespace(), [2, 3])
+
def test_logpages_dump(self):
"""Test the deprecated site.logpages() method using dump mode."""
le = list(self.site.logpages(number=10, dump=True))
--
To view, visit https://gerrit.wikimedia.org/r/375719
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic523845fd2b143be328ee6edc3b3c53783486026
Gerrit-PatchSet: 11
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #4289
Status: Errored
Duration: 4 hours, 28 minutes, and 23 seconds
Commit: 5cf7fb4 (master)
Author: dalba
Message: APISite.logevents: Allow empty list as the namespace parameter
This fixes a regression introduced in ecb4bd67393c158ab9e9406d567dea4208ffd9a8
which was causing test_logpages_dump to fail because the default value for
`namespace` parameter is `[]`.
Change-Id: I9ae4cac160702b12b49ff258e02b0acf5cdbfbad
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/a0ae857f5ba9...5cf7fb42…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/271553771?utm_source=…
--
You can configure recipients for build notifications in your .travis.yml file. See https://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/365256 )
Change subject: Update ChangeLog for release 3.0.20170713
......................................................................
Update ChangeLog for release 3.0.20170713
Change-Id: I099a7e525c0257bdff55a2a50ab4ac6b220a88df
---
M ChangeLog
1 file changed, 105 insertions(+), 0 deletions(-)
Approvals:
Dalba: Looks good to me, approved
jenkins-bot: Verified
diff --git a/ChangeLog b/ChangeLog
index 2e75305..f236a60 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,108 @@
+Release 3.0.20170713 (13 July 2017)
+====================================
+
+Bugfixes (core)
+---------------
+d9d1d02 Manage temporary readonly error (T154011)
+e533425 Unbreak wbGeoShape and WbTabularData (T166362)
+3834b6c Clean up issue with _WbDataPage (T166362)
+b8a9c2f Re-enable xml for WikiStats with py2 (T165830)
+461ffb0 Solve httplib.IncompleteRead exception in eventstreams (T168535)
+c697bfe Only force input_choise if self.always is given (T161483)
+5ed60e0 Add colon when replacing category and file weblink (T127745)
+df097a6 API Request: set uiprop only when ensuring 'userinfo' in meta (T169202)
+39843e8 Fix TestLazyLoginNotExistUsername test for Stewardwiki (T169458)
+
+Improvements (core)
+-------------------
+8b74d51 Introduce the new WbUnknown data type for Wikibase (T165961)
+461f747 djvu.py: add replace_page() and delete_page()
+ee30a68 Build GeoShape and TabularData from shared base class
+9aee0e0 Remove non-breaking spaces when tidying up a link (T130818)
+aef1b24 Replace private mylang variables with mycode in generate_user_files.py
+a5e5a3f FilePage: remove deprecated use of fileUrl
+e236757 Make socket_timeout recalculation reusable (T166539)
+b022690 FilePage.download(): add revision parameter to download arbitrary revision (T166939)
+737b037 Make pywikibot.Error more precise (T166982)
+a0a6d16 Implement pywikibot support for adding thanks to normal revisions (T135409)
+ed61b0f Implement EventStreams (T158943)
+f5f4a60 Add -titleregexnot page filter
+eddcc95 Add exception for -namepace option (T167580)
+4ea8d38 InteractiveReplace: Allow no replacements by default
+633e69a Encode default globe in family file
+3321a40 Add on to pywikibot support for thanking normal revisions (T135409)
+98783dc Add log entry code for thanks log (T135413)
+4be9336 Create superclass for log entries with user targets
+f6b0c7b Use relative reference to class attribute
+d12f60b Allow pywikibot to authenticate against a private wiki (T153903)
+42443c5 Make WbRepresentations hashable (T167827)
+
+Updates (core)
+--------------
+a6337f7 Update linktails
+173becb Update languages_by_size
+be75fb9 Update cross_allowed (global bot wikis group)
+1fbcccb Add atjwiki to wikipedia family file (T168049)
+beacf44 remove closed sites from languages_by_size list
+447b045 Update category_redirect_templates for wikipedia and commons Family
+7945c4a Update logevent type parameter list
+bc01242 Disable cleanUpSectionHeaders on jbo.wiktionary (T168399)
+54b4f90 Add kbpwiki to wikipedia family file (T169216)
+0679efa Remove anarchopedia family out of the framework (T167534)
+
+Bugfixes (scripts)
+------------------
+1d8c1bd Fix ResourceWarning in wikimedia_sites.py (T166459)
+02dbe77 imagecopy.py: page.site is not callable (T167062)
+38d645d weblinkchecker.py: Use https for web.archive.org (T167463)
+ea80e13 redirect.py: Adjust _treat_counter (T167254)
+f24c082 misspelling.py: Fix comparing template name (T125328)
+74f6c36 piper.py: fix bytes/str handling in python3 (T157215)
+fa830d6 Enable exceptions from fixes file (T142324)
+e229a13 revertbot.py: Don't loop over user contribs (T168666)
+e8bdc42 Use correct iterator method in interwikidata.py (T168575)
+86f623f Don't fail TestProofreadPageValidSite.test_url_image (T169542)
+54d24f7 Fix some of the invalid escape sequences (T164575)
+f577676 redirect.py: Handle PageSaveRelatedError exception (T169859)
+48431e7 illustratewikidata: rename page_image key in page_image_free (T169447)
+2684ffc checkimages: format string must be concatenated first (T170087)
+7f2b925 interwikidata: Fix interwiki conflict detection (T168575)
+f645523 Fix ResourceWarning in wikimedia_sites.py with python3 (T166459)
+
+Improvements (scripts)
+----------------------
+3343a7f Port followlive.py to core (T66854)
+92f66ff replace.py: allow to edit modified text
+4ee7fda Catch KeyboardInterrupt and finally print execution time in checkimages.py
+e79af64 proofreadpage.py: raise TypeError, not ValueError
+7c5a078 category.py/tidy: Show page title at the bottom of the output (T166468)
+8355c22 checkimages: Use site.namespaces.FILE.custom_name instead of "File:"
+e2c92e2 weblinkchecker.py: Sorting of carriage returns (T166934)
+a78df27 Stopme is done at exit time by default
+a360417 standardize spamremove.py
+cd5665a replace.py: do not overwrite exceptions given via CLI (T142324)
+280f0a9 Create items with interwiki using interwikidata.py (T168575)
+c0a879b proofreadpage.py: add support for OCR feature (T159655)
+b11c819 Improve maintenance/wikimedia_sites.py
+152d798 proofreadpage.py: Fallback to "html.parser" if "lxml" is not available (T169515)
+09f56d4 Add support for item without link to harvest_template (T64014)
+c586403 Port ndashredir to core (T66875)
+5435fd7 Show different message if the page doesn't have an item (T168575)
+847cbb3 Merge items in interwikidata.py
+
+Updates (scripts)
+-----------------
+b4ab2e1 noreferences.py: update frwiki dict
+c8d1e66 Remove panoramiopicker.py (Panoramio is no longer available)
+b0f0d92 Use bs4 for imageharvest, not BeautifulSoup v3 (T115428)
+4c10c17 blockpageschecker: Add serbian localisation (T167599)
+b398a93 Update Czech (cs) summary in standardize_interwiki.py (T168798)
+2b882ed Update Czech category name in category_redirect.py
+f088a04 Remove anarchopedia from wikimedia_sites.py maintenance script (T167534)
+9b3a556 Add a French localisation for clean_sandbox
+77f42c0 Add Hindi (hi) summary in standardize_interwiki.py
+
+
Release 3.0.20170521 (21 Mai 2017)
====================================
--
To view, visit https://gerrit.wikimedia.org/r/365256
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I099a7e525c0257bdff55a2a50ab4ac6b220a88df
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375710 )
Change subject: APISite.logevents: Allow empty list as the namespace parameter
......................................................................
APISite.logevents: Allow empty list as the namespace parameter
This fixes a regression introduced in ecb4bd67393c158ab9e9406d567dea4208ffd9a8
which was causing test_logpages_dump to fail because the default value for
`namespace` parameter is `[]`.
Change-Id: I9ae4cac160702b12b49ff258e02b0acf5cdbfbad
---
M pywikibot/site.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 4a4dba6..2b81e4d 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -4544,7 +4544,7 @@
legen.request["leend"] = end
if reverse:
legen.request["ledir"] = "newer"
- if namespace is not None:
+ if namespace or namespace == 0:
legen.request["lenamespace"] = namespace
if tag:
# Supported in version 1.16+; earlier sites will cause APIError
--
To view, visit https://gerrit.wikimedia.org/r/375710
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9ae4cac160702b12b49ff258e02b0acf5cdbfbad
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375606 )
Change subject: APISite.logevents: Do not ignore the namespace if it is 0
......................................................................
APISite.logevents: Do not ignore the namespace if it is 0
Bug: T174875
Change-Id: Icd3ea445a08fb88c104663952c4d9afd2a520739
---
M pywikibot/site.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
Zoranzoki21: Looks good to me, but someone else must approve
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 67a782e..4a4dba6 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -4544,7 +4544,7 @@
legen.request["leend"] = end
if reverse:
legen.request["ledir"] = "newer"
- if namespace:
+ if namespace is not None:
legen.request["lenamespace"] = namespace
if tag:
# Supported in version 1.16+; earlier sites will cause APIError
--
To view, visit https://gerrit.wikimedia.org/r/375606
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icd3ea445a08fb88c104663952c4d9afd2a520739
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zoranzoki21 <dori.gord(a)orion.rs>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/362410 )
Change subject: [DOC] Differentiate between generators and filters in -help
......................................................................
[DOC] Differentiate between generators and filters in -help
- also sort filter parameters
Bug: T167581
Change-Id: I7054328600292f3306b000d61151d32fa7293f08
---
M pywikibot/bot.py
M pywikibot/pagegenerators.py
2 files changed, 59 insertions(+), 51 deletions(-)
Approvals:
Dalba: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 500fb5d..baa57ac 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -995,7 +995,9 @@
module_name = "no_module"
globalHelp = u'''
-Global arguments available for all bots:
+GLOBAL OPTIONS
+==============
+(Global arguments available for all bots)
-dir:PATH Read the bot's configuration data from directory given by
PATH, instead of from the default directory.
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 32c1c22..f775d54 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -66,9 +66,8 @@
# a generator
parameterHelp = """\
-
--catfilter Filter the page generator to only yield pages in the
- specified category. See -cat for argument format.
+GENERATOR OPTIONS
+=================
-cat Work on all pages which are in a specific category.
Argument can also be given as "-cat:categoryname" or
@@ -144,37 +143,12 @@
In some cases it must be written as -logevents:"move,Usr,20"
--namespaces Filter the page generator to only yield pages in the
--namespace specified namespaces. Separate multiple namespace
--ns numbers or names with commas.
- Examples:
-
- -ns:0,2,4
- -ns:Help,MediaWiki
-
- You may use a preleading "not" to exclude the namespace.
- Examples:
- -ns:not:2,3
- -ns:not:Help,File
-
- If used with -newpages/-random/-randomredirect,
- -namespace/ns must be provided before
- -newpages/-random/-randomredirect.
- If used with -recentchanges, efficiency is improved if
- -namespace/ns is provided before -recentchanges.
-
- If used with -start, -namespace/ns shall contain only one
- value.
-
-interwiki Work on the given page and all equivalent pages in other
languages. This can, for example, be used to fight
multi-site spamming.
Attention: this will cause the bot to modify
pages on several wiki sites, this is not well tested,
so check your edits!
-
--limit:n When used with any other argument that specifies a set
- of pages, work on no more than n pages in total.
-links Work on all pages that are linked from a certain page.
Argument can also be given as "-links:linkingpagetitle".
@@ -225,22 +199,6 @@
default value is start:!
-prefixindex Work on pages commencing with a common prefix.
-
--subpage:n Filters pages to only those that have depth n
- i.e. a depth of 0 filters out all pages that are subpages,
- and a depth of 1 filters out all pages that are subpages of
- subpages.
-
--titleregex A regular expression that needs to match the article title
- otherwise the page won't be returned.
- Multiple -titleregex:regexpr can be provided and the page
- will be returned if title is matched by any of the regexpr
- provided.
- Case insensitive regular expressions will be used and
- dot matches any character.
-
--titleregexnot Like -titleregex, but return the page only if the regular
- expression does not match.
-transcludes Work on all pages that use a certain template.
Argument can also be given as "-transcludes:Title".
@@ -315,6 +273,13 @@
"-pageid:pageid1,pageid2,." or "-pageid:'pageid1|pageid2|..'"
and supplied multiple times for multiple pages.
+
+FILTER OPTIONS
+==============
+
+-catfilter Filter the page generator to only yield pages in the
+ specified category. See -cat generator for argument format.
+
-grep A regular expression that needs to match the article
otherwise the page won't be returned.
Multiple -grep:regexpr can be provided and the page will
@@ -323,11 +288,32 @@
Case insensitive regular expressions will be used and
dot matches any character, including a newline.
--ql Filter pages based on page quality.
- This is only applicable if contentmodel equals
- 'proofread-page', otherwise has no effects.
- Valid values are in range 0-4.
- Multiple values can be comma-separated.
+-intersect Work on the intersection of all the provided generators.
+
+-limit:n When used with any other argument that specifies a set
+ of pages, work on no more than n pages in total.
+
+-namespaces Filter the page generator to only yield pages in the
+-namespace specified namespaces. Separate multiple namespace
+-ns numbers or names with commas.
+ Examples:
+
+ -ns:0,2,4
+ -ns:Help,MediaWiki
+
+ You may use a preleading "not" to exclude the namespace.
+ Examples:
+ -ns:not:2,3
+ -ns:not:Help,File
+
+ If used with -newpages/-random/-randomredirect generators,
+ -namespace/ns must be provided before
+ -newpages/-random/-randomredirect.
+ If used with -recentchanges generator, efficiency is
+ improved if -namespace/ns is provided before -recentchanges.
+
+ If used with -start generator, -namespace/ns shall contain
+ only one value.
-onlyif A claim the page needs to contain, otherwise the item won't
be returned.
@@ -347,7 +333,27 @@
be returned.
For usage and examples, see -onlyif above.
--intersect Work on the intersection of all the provided generators.
+-ql Filter pages based on page quality.
+ This is only applicable if contentmodel equals
+ 'proofread-page', otherwise has no effects.
+ Valid values are in range 0-4.
+ Multiple values can be comma-separated.
+
+-subpage:n Filters pages to only those that have depth n
+ i.e. a depth of 0 filters out all pages that are subpages,
+ and a depth of 1 filters out all pages that are subpages of
+ subpages.
+
+-titleregex A regular expression that needs to match the article title
+ otherwise the page won't be returned.
+ Multiple -titleregex:regexpr can be provided and the page
+ will be returned if title is matched by any of the regexpr
+ provided.
+ Case insensitive regular expressions will be used and
+ dot matches any character.
+
+-titleregexnot Like -titleregex, but return the page only if the regular
+ expression does not match.
"""
docuReplacements = {'¶ms;': parameterHelp}
--
To view, visit https://gerrit.wikimedia.org/r/362410
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7054328600292f3306b000d61151d32fa7293f08
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375491 )
Change subject: Add Config in clean_sandbox.py for hi.wiki
......................................................................
Add Config in clean_sandbox.py for hi.wiki
Bug: T174844
Change-Id: Ic397ff01a11a1b9653bd259e18aa39945c66a2cd
---
M scripts/clean_sandbox.py
1 file changed, 1 insertion(+), 0 deletions(-)
Approvals:
Mpaa: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/clean_sandbox.py b/scripts/clean_sandbox.py
index 47457fc..e5f1bf8 100755
--- a/scripts/clean_sandbox.py
+++ b/scripts/clean_sandbox.py
@@ -72,6 +72,7 @@
'fi': u'{{subst:Hiekka}}',
'fr': '{{subst:Préchargement pour Bac à sable}}',
'he': u'{{ארגז חול}}\n<!-- נא לערוך מתחת לשורה זו בלבד, תודה. -->',
+ 'hi': '{{User sandbox}}\n<!-- कृप्या इस लाइन के नीचे सम्पादन करे। -->',
'id': u'{{Bakpasir}}\n<!-- Uji coba dilakukan di baris di bawah ini -->',
'it': '{{sandbox}}'
'<!-- Scrivi SOTTO questa riga senza cancellarla. Grazie. -->',
--
To view, visit https://gerrit.wikimedia.org/r/375491
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic397ff01a11a1b9653bd259e18aa39945c66a2cd
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Jayprakash12345 <0freerunning(a)gmail.com>
Gerrit-Reviewer: Framawiki <framawiki(a)tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/375376 )
Change subject: harvest_template.py: Do not use `list.clear` which is python 3 only
......................................................................
harvest_template.py: Do not use `list.clear` which is python 3 only
Bug: T174785
Change-Id: Ia4270361e5e53742bc4472493a82b95e7a757aa1
---
M scripts/harvest_template.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
Strainu: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py
index 81fd492..7303524 100755
--- a/scripts/harvest_template.py
+++ b/scripts/harvest_template.py
@@ -353,7 +353,7 @@
if complete:
handler = PropertyOptionHandler(**current_args[2])
fields[current_args[0]] = (current_args[1], handler)
- current_args.clear()
+ del current_args[:]
else:
current_args.append(arg)
if len(current_args) == 2:
--
To view, visit https://gerrit.wikimedia.org/r/375376
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia4270361e5e53742bc4472493a82b95e7a757aa1
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Strainu <wiki(a)strainu.ro>
Gerrit-Reviewer: jenkins-bot <>
