Build Update for jayvdb/pywikibot-core
-------------------------------------
Build: #160
Status: Failed
Duration: 37 minutes and 20 seconds
Commit: bd80770 (tests-reenable-site-tests)
Author: John Vandenberg
Message: Re-enable commented out site tests
- allpages tests were commented out for performance reasons.
0da0521
- allcategories and allimages tests commented out due to bug, now fixed.
Bug: T60941
Change-Id: I13be662e1be1f90109c8a19e6e9b13d65419255a
View the changeset: https://github.com/jayvdb/pywikibot-core/compare/e10bd6e9e937^...bd8077008a…
View the full build log and details: https://travis-ci.org/jayvdb/pywikibot-core/builds/43674767
--
You can configure recipients for build notifications in your .travis.yml file. See http://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged.
Change subject: Add LogeventsPageGenerator, LogpagesPageGenerator
......................................................................
Add LogeventsPageGenerator, LogpagesPageGenerator
LogpagesPageGenerator ported from compat to be backwards compatible.
LogpagesPageGenerator is marked as deprecated. number, mode and repeat
are all deprecated: number was changed to total, mode was changed to
logtype, and repeat was removed all-together.
LogeventsPageGenerator is a new generator providing the same
functionality, with updated parameters for core.
Using title() on the results of logitems() returns a Page
(this DOES return a Page but it seems to be a bug. See issue T78125).
Added namespace argument to logevents() which is passed to the request
as "lenamespace".
The command line arguments were also ported from compat using the same
rules. Added TODO for validating log type. This should raise an error.
Unittests in tests/pagegenerators_test.TestLogpagesFactoryGenerator were
added for LogeventsPageGenerator and the command line args. Added
user = True for this test case for testing the use of users in the
LogpagesPageGenerator command line args. These tests have a TODO
indicating the need for a way to check if the logs belong to that user.
Bug: T76555
Change-Id: Ide811094e4d50d5a8375a97bf8ead62eba647814
---
M pywikibot/pagegenerators.py
M pywikibot/site.py
M tests/pagegenerators_tests.py
3 files changed, 146 insertions(+), 1 deletion(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 00e0f11..e5c0e31 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -31,6 +31,7 @@
import pywikibot
from pywikibot import date, config, i18n
from pywikibot.tools import (
+ deprecated,
deprecated_args,
DequeGenerator,
intersect_generators,
@@ -85,6 +86,19 @@
-search Work on all pages that are found in a MediaWiki search
across all namespaces.
+
+-<logevent>log Work on articles that were on a specified special:log.
+ You have options for every type of logs given by the
+ <logevent> parameter which could be one of the following:
+ block, protect, rights, delete, upload, move, import,
+ patrol, merge, suppress, review, stable, gblblock,
+ renameuser, globalauth, gblrights, abusefilter, newusers
+ Examples:
+ -movelog gives 500 pages from move log (should be redirects)
+ -deletelog:10 gives 10 pages from deletion log
+ -protect:Dummy gives 500 pages from protect by user Dummy
+ -patrol:Dummy;20 gives 20 pages patroled by user Dummy
+ In some cases this must be written as -patrol:"Dummy;20"
-namespaces Filter the page generator to only yield pages in the
-namespace specified namespaces. Separate multiple namespace
@@ -595,6 +609,29 @@
elif arg.startswith('-intersect'):
self.intersect = True
return True
+ elif arg.startswith('-'):
+ mode, log, user = arg.partition('log')
+ # exclude -log, -nolog
+ if log == 'log' and mode not in ['-', '-no']:
+ total = 500
+ if not user:
+ user = None
+ else:
+ try:
+ total = int(user[1:])
+ user = None
+ except ValueError:
+ user = user[1:]
+ result = user.split(';')
+ user = result[0]
+ try:
+ total = int(result[1])
+ except (ValueError, IndexError):
+ pywikibot.error(
+ u'Value specified after ";" not an int.')
+ return False
+ # TODO: Check if mode[1:] is one of the allowed log types
+ gen = LogeventsPageGenerator(mode[1:], user, total=total)
if gen:
self.gens.append(gen)
@@ -665,6 +702,55 @@
content=content)
+@deprecated_args(number="total", mode="logtype", repeat=None)
+def LogeventsPageGenerator(logtype=None, user=None, site=None,
+ namespace=0, total=None):
+ """
+ Generate Pages for specified modes of logevents.
+
+ @param logtype: Mode of logs to retrieve
+ @type logtype: basestring
+ @param user: User of logs retrieved
+ @type user: basestring
+ @param site: Site for generator results
+ @type site: L{pywikibot.site.BaseSite}
+ @param namespace: Namespace to retrieve logs from
+ @type namespace: int
+ @param total: Maximum number of pages to retrieve in total
+ @type total: int
+ """
+ if site is None:
+ site = pywikibot.Site()
+ for entry in site.logevents(total=total, logtype=logtype,
+ user=user, namespace=namespace):
+ yield entry.title()
+
+
+@deprecated("LogeventsPageGenerator")
+@deprecated_args(number="total", mode="logtype", repeat=None)
+def LogpagesPageGenerator(total=500, logtype='', user=None,
+ site=None, namespace=[]):
+ """
+ Generate Pages for specified modes of logevents.
+
+ This is the backwards compatible one.
+ See LogeventsPageGenerator
+
+ @param mode: Mode of logs to retrieve
+ @type mode: basestring
+ @param user: User of logs retrieved
+ @type user: basestring
+ @param site: Site for generator results
+ @type site: L{pywikibot.site.BaseSite}
+ @param namespace: Namespace to retrieve logs from
+ @type namespace: int
+ @param total: Maximum number of pages to retrieve in total
+ @type total: int
+ """
+ return LogeventsPageGenerator(total=total, logtype=logtype, user=user,
+ site=site, namespace=namespace)
+
+
@deprecated_args(number="total", namespace="namespaces", repeat=None)
def NewpagesPageGenerator(get_redirect=False, site=None,
namespaces=[0, ], step=None, total=None):
diff --git a/pywikibot/site.py b/pywikibot/site.py
index c2d9014..aea95ca 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -3388,7 +3388,7 @@
total=total, g_content=content, **iuargs)
return iugen
- def logevents(self, logtype=None, user=None, page=None,
+ def logevents(self, logtype=None, user=None, page=None, namespace=None,
start=None, end=None, reverse=False, step=None, total=None):
"""Iterate all log entries.
@@ -3398,6 +3398,7 @@
"patrol", "merge")
@param user: only iterate entries that match this user name
@param page: only iterate entries affecting this page
+ @param namespace: namespace to retrieve logevents from
@param start: only iterate entries from and after this Timestamp
@param end: only iterate entries up to and through this Timestamp
@param reverse: if True, iterate oldest entries first (default: newest)
@@ -3420,6 +3421,8 @@
legen.request["leend"] = str(end)
if reverse:
legen.request["ledir"] = "newer"
+ if namespace:
+ legen.request["lenamespace"] = namespace
return legen
def recentchanges(self, start=None, end=None, reverse=False,
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 9e19bde..32f3fda 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -392,6 +392,62 @@
self.assertPagesInNamespaces(gen, set([1, 3]))
+class TestLogeventsFactoryGenerator(DefaultSiteTestCase):
+
+ """Test GeneratorFactory with pagegenerators.LogeventsPageGenerator."""
+
+ user = True
+
+ @unittest.expectedFailure
+ def test_logevents_parse(self):
+ gf = pagegenerators.GeneratorFactory()
+ self.assertFalse(gf.handleArg("-log"))
+ self.assertFalse(gf.handleArg("-log:text_here"))
+ # TODO: Throw an error for incorrect logtypes
+ self.assertRaises(gf.handleArg("-this_will_never_be_a_typelog"),
+ Exception)
+
+ def test_logevents_default(self):
+ gf = pagegenerators.GeneratorFactory(site=self.site)
+ self.assertTrue(gf.handleArg('-newuserslog'))
+ gen = gf.getCombinedGenerator()
+ pages = set(gen)
+ self.assertLessEqual(len(pages), 500)
+ self.assertTrue(all(isinstance(item, pywikibot.Page) for item in pages))
+
+ def test_logevents_default_multi(self):
+ gf = pagegenerators.GeneratorFactory(site=self.site)
+ self.assertTrue(gf.handleArg('-newuserslog:10'))
+ gen = gf.getCombinedGenerator()
+ pages = set(gen)
+ self.assertLessEqual(len(pages), 10)
+ self.assertTrue(all(isinstance(item, pywikibot.Page) for item in pages))
+
+ def test_logevents_ns(self):
+ gf = pagegenerators.GeneratorFactory(site=self.site)
+ gf.handleArg('-ns:1')
+ gf.handleArg('-newuserslog:10')
+ gen = gf.getCombinedGenerator()
+ self.assertPagesInNamespaces(gen, 1)
+ self.assertTrue(all(isinstance(item, pywikibot.Page) for item in gen))
+
+ def test_logevents_user_multi(self):
+ gf = pagegenerators.GeneratorFactory(site=self.site)
+ user = self.get_site().user()
+ self.assertTrue(gf.handleArg('-newuserslog:' + user + ';10'))
+ gen = gf.getCombinedGenerator()
+ pages = set(gen)
+
+ if not pages:
+ raise unittest.SkipTest('No user creation log entries for ' + user)
+
+ # TODO: Check if the pages generated correspond to the user
+ # (no easy way of checking from pages)
+
+ self.assertLessEqual(len(pages), 10)
+ self.assertTrue(all(isinstance(item, pywikibot.Page) for item in pages))
+
+
class PageGeneratorIntersectTestCase(DefaultSiteTestCase,
GeneratorIntersectTestCase):
--
To view, visit https://gerrit.wikimedia.org/r/178773
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ide811094e4d50d5a8375a97bf8ead62eba647814
Gerrit-PatchSet: 12
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Daviskr <davis(a)daviskr.com>
Gerrit-Reviewer: Daviskr <davis(a)daviskr.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Preserve HTTP worker state after exception
......................................................................
Preserve HTTP worker state after exception
threadedhttp.Http.request set self.follow_redirects=False before calling its
parent method, and reset it afterwards. However if there was an exception
in the parent method, self.follow_redirects was not reset and changed
the behaviour of subsequent requests handled by the HTTP worker.
Change-Id: I1b917fbcea4ad51a958ec19faea4f2618662f6a8
---
M pywikibot/comms/threadedhttp.py
M tests/http_tests.py
2 files changed, 20 insertions(+), 1 deletion(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py
index af9ce82..8de4213 100644
--- a/pywikibot/comms/threadedhttp.py
+++ b/pywikibot/comms/threadedhttp.py
@@ -230,7 +230,8 @@
except Exception as e: # what types?
# return exception instance to be retrieved by the calling thread
return e
- self.follow_redirects = follow_redirects
+ finally:
+ self.follow_redirects = follow_redirects
# return connection to pool
self.connection_pool.push_connection(conn_key,
diff --git a/tests/http_tests.py b/tests/http_tests.py
index abef94a..fdcd478 100644
--- a/tests/http_tests.py
+++ b/tests/http_tests.py
@@ -114,6 +114,24 @@
http.fetch,
uri='http://getstatuscode.com/504')
+ def test_follow_redirects(self):
+ """Test follow 301 redirects after an exception works correctly."""
+ # to be effective, this exception should be raised in httplib2
+ self.assertRaises(Exception,
+ http.fetch,
+ uri='invalid://url')
+
+ # The following will redirect from ' ' -> '_', and maybe to https://
+ r = http.fetch(uri='http://en.wikipedia.org/wiki/Main%20Page')
+ self.assertEqual(r.status, 200)
+ self.assertIn('//en.wikipedia.org/wiki/Main_Page',
+ r.response_headers['content-location'])
+
+ r = http.fetch(uri='http://www.gandi.eu')
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.response_headers['content-location'],
+ 'http://www.gandi.net')
+
class ThreadedHttpTestCase(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/178789
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1b917fbcea4ad51a958ec19faea4f2618662f6a8
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot <>