jenkins-bot has submitted this change and it was merged.
Change subject: Do not print passwords to console
......................................................................
Do not print passwords to console
The config parser can be run on the command line to print all values.
This routine is useful for debugging, however as a general rule passwords
should never be printed to the console.
Change-Id: I41bd9d91bc5a858cf7eadbf43b79df850dcb658c
---
M pywikibot/config2.py
1 file changed, 14 insertions(+), 1 deletion(-)
Approvals:
Ladsgroup: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 7193aef..9fa0a6a 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -21,6 +21,8 @@
# variables that are intended only for internal use and not to be exported
# to other modules.
+_private_values = ['authenticate', 'proxy', 'db_password']
+
# ############# ACCOUNT SETTINGS ##############
# The family of sites we are working on. wikipedia.py will import
@@ -759,7 +761,18 @@
if not type(globals()[_name]) in [types.FunctionType,
types.ModuleType]:
if _all or _glv[_name] != globals()[_name]:
- print("%s=%s" % (_name, repr(globals()[_name])))
+ _value = globals()[_name]
+ if _name in _private_values and _value:
+ if isinstance(_value, dict):
+ _value = '{ ...xxxxxxxx... }'
+ elif hasattr(_value, '__dict__'):
+ _value = '%s( ...xxxxxxxx... )' % \
+ _value.__class__.__name__
+ else:
+ _value = repr('xxxxxxxx')
+ else:
+ _value = repr(_value)
+ print("%s=%s" % (_name, _value))
# cleanup all locally-defined variables
for __var in list(globals().keys()):
--
To view, visit https://gerrit.wikimedia.org/r/141432
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I41bd9d91bc5a858cf7eadbf43b79df850dcb658c
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Betacommand <betacommand(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: archivebot: use canonical title for calculating salt
......................................................................
archivebot: use canonical title for calculating salt
Change-Id: I0215d1f75755e65ad552c57f107e3a5252d7fab7
---
M scripts/archivebot.py
1 file changed, 7 insertions(+), 1 deletion(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/archivebot.py b/scripts/archivebot.py
index 0e8d54b..018c225 100644
--- a/scripts/archivebot.py
+++ b/scripts/archivebot.py
@@ -510,6 +510,8 @@
if not arg.startswith('-'):
args.append(arg)
+ site = pywikibot.Site()
+
if calc:
if not salt:
pywikibot.showHelp()
@@ -517,6 +519,11 @@
'NOTE: you must specify a salt to calculate a key using '
'-salt:SALT option.')
return
+ page = pywikibot.Page(site, calc)
+ if page.exists():
+ calc = page.title()
+ else:
+ pywikibot.output(u'NOTE: the specified page "%s" does not (yet) exist.' % calc)
s = md5()
s.update(salt + '\n')
s.update(calc + '\n')
@@ -531,7 +538,6 @@
pywikibot.output(u'NOTE: you must specify a template to run the bot.')
return
- site = pywikibot.Site()
for a in args:
pagelist = []
if not filename and not pagename:
--
To view, visit https://gerrit.wikimedia.org/r/135303
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0215d1f75755e65ad552c57f107e3a5252d7fab7
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Whym <whym(a)whym.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Whym <whym(a)whym.org>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Remove unnecessary tuple return type
......................................................................
Remove unnecessary tuple return type
The tuple return type of addCommonscat() has been unnecessary since 2010.
https://www.mediawiki.org/wiki/Special:Code/pywikipedia/8548
Returned (status, always) were unused variables.
Change-Id: I767c00ea8b500dc955c1f80b626a076c1c330510
---
M scripts/commonscat.py
1 file changed, 5 insertions(+), 6 deletions(-)
Approvals:
Guoguo12: Looks good to me, but someone else must approve
Ladsgroup: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/commonscat.py b/scripts/commonscat.py
index e0f1b9a..de28ab2 100755
--- a/scripts/commonscat.py
+++ b/scripts/commonscat.py
@@ -262,8 +262,7 @@
pywikibot.output(u'Page %s is a disambiguation. Skipping.'
% page.title(asLink=True))
else:
- (status, always) = self.addCommonscat(page)
- return
+ self.addCommonscat(page)
def load(self, page):
""" Load the given page, do some changes, and save it. """
@@ -369,14 +368,14 @@
# The current commonscat link is good
pywikibot.output(u'Commonscat link at %s to Category:%s is ok'
% (page.title(), currentCommonscatTarget))
- return (True, self.always)
+ return True
elif checkedCommonscatTarget != u'':
# We have a new Commonscat link, replace the old one
self.changeCommonscat(page, currentCommonscatTemplate,
currentCommonscatTarget,
primaryCommonscat,
checkedCommonscatTarget, LinkText, Note)
- return (True, self.always)
+ return True
else:
#Commonscat link is wrong
commonscatLink = self.findCommonscatLink(page)
@@ -403,8 +402,8 @@
self.summary,
None, None,
self.always)
- return (True, self.always)
- return (True, self.always)
+ return True
+ return True
def changeCommonscat(self, page=None, oldtemplate=u'', oldcat=u'',
newtemplate=u'', newcat=u'', linktitle=u'',
--
To view, visit https://gerrit.wikimedia.org/r/141370
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I767c00ea8b500dc955c1f80b626a076c1c330510
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Betacommand <betacommand(a)gmail.com>
Gerrit-Reviewer: Guoguo12 <Guoguo12(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Nullzero <nullzero.free(a)gmail.com>
Gerrit-Reviewer: Pyfisch <pyfisch(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: devunt <devunt(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Bug 65966: fix GoogleSearchPageGenerator
......................................................................
Bug 65966: fix GoogleSearchPageGenerator
Use pypi package 'google' for GoogleSearchPageGenerator
and print a 'terms of use' warning for each query.
A few pep257 fixes in nearby code.
Change-Id: Ieaa92eead313a7948879c7668c46fbf15eeb396d
---
M pywikibot/pagegenerators.py
1 file changed, 49 insertions(+), 82 deletions(-)
Approvals:
John Vandenberg: Looks good to me, but someone else must approve
Ladsgroup: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 98e292b..3148e32 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -1223,7 +1223,14 @@
class YahooSearchPageGenerator:
- """ To use this generator, install pYsearch """
+ """
+ Page generator using Yahoo! search results.
+
+ To use this generator, you need to install the package 'pYsearch'.
+ https://pypi.python.org/pypi/pYsearch
+
+ To use this generator, install pYsearch
+ """
# values larger than 100 fail
def __init__(self, query=None, count=100, site=None):
@@ -1234,7 +1241,15 @@
self.site = site
def queryYahoo(self, query):
- from yahoo.search.web import WebSearch
+ """ Perform a query using python package 'pYsearch'. """
+ try:
+ from yahoo.search.web import WebSearch
+ except ImportError:
+ pywikibot.error("ERROR: generator YahooSearchPageGenerator "
+ "depends on package 'pYsearch'.\n"
+ "To install, please run: pip install pYsearch")
+ exit(1)
+
srch = WebSearch(config.yahoo_appid, query=query, results=self.count)
dom = srch.get_results()
results = srch.parse_results(dom)
@@ -1257,11 +1272,16 @@
class GoogleSearchPageGenerator:
"""
- To use this generator, you must install the pyGoogle module from
- http://pygoogle.sf.net/ and get a Google Web API license key from
- http://www.google.com/apis/index.html . The google_key must be set to your
- license key in your configuration.
+ Page generator using Google search results.
+ To use this generator, you need to install the package 'google'.
+ https://pypi.python.org/pypi/google
+
+ This package has been available since 2010, hosted on github
+ since 2012, and provided by pypi since 2013.
+
+ As there are concerns about Google's Terms of Service, this
+ generator prints a warning for each query.
"""
def __init__(self, query=None, site=None):
@@ -1271,83 +1291,30 @@
self.site = site
def queryGoogle(self, query):
- #########
- # Google's "Terms of service"
- # (see 5.3, http://www.google.com/accounts/TOS?loc=US)
- for url in self.queryViaSoapApi(query):
+ """
+ Perform a query using python package 'google'.
+
+ The terms of service as at June 2014 give two conditions that
+ may apply to use of search:
+ 1. Dont access [Google Services] using a method other than
+ the interface and the instructions that [they] provide.
+ 2. Don't remove, obscure, or alter any legal notices
+ displayed in or along with [Google] Services.
+
+ Both of those issues should be managed by the package 'google',
+ however pywikibot will at least ensure the user sees the TOS
+ in order to comply with the second condition.
+ """
+ try:
+ import google
+ except ImportError:
+ pywikibot.error("ERROR: generator GoogleSearchPageGenerator "
+ "depends on package 'google'.\n"
+ "To install, please run: pip install google.")
+ exit(1)
+ pywikibot.warning('Please read http://www.google.com/accounts/TOS')
+ for url in google.search(query):
yield url
-
- def queryViaSoapApi(self, query):
- import google
- google.LICENSE_KEY = config.google_key
- offset = 0
- estimatedTotalResultsCount = None
- while not estimatedTotalResultsCount or \
- offset < estimatedTotalResultsCount:
- while True:
- # Google often yields 502 errors.
- try:
- pywikibot.output(u'Querying Google, offset %i' % offset)
- data = google.doGoogleSearch(query, start=offset,
- filter=False)
- break
- except KeyboardInterrupt:
- raise
- except:
- # SOAPpy.Errors.HTTPError or SOAP.HTTPError
- # (502 Bad Gateway) can happen here, depending on the module
- # used. It's not easy to catch this properly because
- # pygoogle decides which one of the soap modules to use.
- pywikibot.output(u"An error occured. "
- u"Retrying in 10 seconds...")
- time.sleep(10)
- continue
-
- for result in data.results:
- yield result.URL
- # give an estimate of pages to work on, but only once.
- if not estimatedTotalResultsCount:
- pywikibot.output(u'Estimated total result count: %i pages.'
- % data.meta.estimatedTotalResultsCount)
- estimatedTotalResultsCount = data.meta.estimatedTotalResultsCount
- offset += 10
-
-# ############
-# commented out because it is probably not in compliance with Google's
-# "Terms of service" (see 5.3, http://www.google.com/accounts/TOS?loc=US)
-#
-# def queryViaWeb(self, query):
-# """
-# Google has stopped giving out API license keys, and sooner or later
-# they will probably shut down the service.
-# This is a quick and ugly solution: we just grab the search results from
-# the normal web interface.
-# """
-# linkR = re.compile(r'<a href="([^>"]+?)" class=l>', re.IGNORECASE)
-# offset = 0
-#
-# while True:
-# pywikibot.output("Google: Querying page %d" % (offset / 100 + 1))
-# address = "http://www.google.com/search?q=%s&num=100&hl=en&start=%d" \
-# % (urllib.quote_plus(query), offset)
-# # we fake being Firefox because Google blocks unknown browsers
-# request = urllib2.Request(
-# address, None,
-# {'User-Agent':
-# 'Mozilla/5.0 (X11; U; Linux i686; de; rv:1.8) Gecko/20051128 '
-# 'SUSE/1.5-0.1 Firefox/1.5'})
-# urlfile = urllib2.urlopen(request)
-# page = urlfile.read()
-# urlfile.close()
-# for url in linkR.findall(page):
-# yield url
-#
-# # Is there a "Next" link for next page of results?
-# if "<div id=nn>" in page:
-# offset += 100 # Yes, go to next page of results.
-# else:
-# return
-# ###########
def __iter__(self):
# restrict query to local site
--
To view, visit https://gerrit.wikimedia.org/r/139061
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ieaa92eead313a7948879c7668c46fbf15eeb396d
Gerrit-PatchSet: 6
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
