jenkins-bot has submitted this change and it was merged.
Change subject: Function to check whether a site has a certain extension loaded
......................................................................
Function to check whether a site has a certain extension loaded
Change-Id: I0fc57f974b19da5d929f119f759f5cd6d56c5e43
---
M pywikibot/site.py
1 file changed, 9 insertions(+), 1 deletion(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 2775f2b..43ec1df 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -1086,7 +1086,7 @@
site=self,
action="query",
meta="siteinfo",
- siprop="general|namespaces|namespacealiases"
+ siprop="general|namespaces|namespacealiases|extensions"
)
try:
sidata = sirequest.submit()
@@ -1127,6 +1127,14 @@
# this is a less preferred form so it goes at the end
self._namespaces[int(item['id'])].append(item["*"])
+ def hasExtension(self, name):
+ if not 'extensions' in self.siteinfo:
+ return NotImplementedError("Feature 'hasExtension' only available in MW 1.14+")
+ for ext in self.siteinfo['extensions']:
+ if ext['name'] == name:
+ return True
+ return False
+
@property
def siteinfo(self):
"""Site information dict."""
--
To view, visit https://gerrit.wikimedia.org/r/82791
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0fc57f974b19da5d929f119f759f5cd6d56c5e43
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: Make return value of defaultsort more explicit
......................................................................
Make return value of defaultsort more explicit
In a seperate commit to prevent a big rebase mess.
Change-Id: I5300844e8c992014ecb13e9eaa77cdc56e1678b5
---
M pywikibot/page.py
1 file changed, 3 insertions(+), 2 deletions(-)
Approvals:
Legoktm: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 4147b4f..7bbc327 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -380,9 +380,10 @@
def defaultsort(self, force=False):
"""
- Returns the value of {{DEFAULTSORT:}} magic word
+ Returns the value of {{DEFAULTSORT:}} magic word or None if no
+ defaultsort has been defined.
@param force: force updating from the live site
- @return: unicode
+ @return: unicode or None
"""
return self.properties(force=force).get('defaultsort')
--
To view, visit https://gerrit.wikimedia.org/r/83005
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5300844e8c992014ecb13e9eaa77cdc56e1678b5
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: If we already have pageproperties, use it for ItemPage.fromPage
......................................................................
If we already have pageproperties, use it for ItemPage.fromPage
The Wikibase Id is stored in the pageproperties, so lets use
it if we can.
Change-Id: I1a4a7d4235bd4e3717f0e6248ab90f10460f264b
---
M pywikibot/page.py
1 file changed, 6 insertions(+), 0 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 4147b4f..07fe099 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -2526,8 +2526,14 @@
def fromPage(cls, page):
"""
Get the ItemPage based on a Page that links to it
+ @param page: Page
+ @return: ItemPage
"""
repo = page.site.data_repository()
+ if hasattr(page, '_pageprops') and page.properties().get('wikibase_item'):
+ # If we have already fetched the pageprops for something else,
+ # we already have the id, so use it
+ return cls(repo, page.properties().get('wikibase_item'))
i = cls(repo, 'null')
del i.id
i._site = page.site
--
To view, visit https://gerrit.wikimedia.org/r/82775
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1a4a7d4235bd4e3717f0e6248ab90f10460f264b
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: Support for fetching page properties
......................................................................
Support for fetching page properties
Added a convenience function for fetching
the defaultsort value.
Change-Id: I0752f1560e0e93cba0e01a9a1ea834e35f8a725c
---
M pywikibot/data/api.py
M pywikibot/page.py
M pywikibot/site.py
3 files changed, 35 insertions(+), 0 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 2b23fb4..17f4db9 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -989,6 +989,9 @@
coords.append(coord)
page._coords = coords
+ if "pageprops" in pagedict:
+ page._pageprops = pagedict['pageprops']
+
if __name__ == "__main__":
from pywikibot import Site, logging
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 59108e3..4147b4f 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -368,6 +368,24 @@
if hasattr(self, "_text"):
del self._text
+ def properties(self, force=False):
+ """
+ Returns the various page properties stored for a page
+ @param force: force updating from the live site
+ @return: dict
+ """
+ if not hasattr(self, '_pageprops') or force:
+ self.site.loadpageprops(self)
+ return self._pageprops
+
+ def defaultsort(self, force=False):
+ """
+ Returns the value of {{DEFAULTSORT:}} magic word
+ @param force: force updating from the live site
+ @return: unicode
+ """
+ return self.properties(force=force).get('defaultsort')
+
def expand_text(self, refresh=False):
"""Return the page text with all templates expanded."""
if not hasattr(self, "_expanded_text") or (self._expanded_text is None) or refresh:
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 1c0b144..5050f78 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -1266,6 +1266,20 @@
continue
api.update_page(page, pageitem)
+ def loadpageprops(self, page):
+ title = page.title(withSection=False)
+ query = self._generator(api.PropertyGenerator,
+ type_arg="pageprops",
+ titles=title.encode(self.encoding()),
+ )
+ for pageitem in query:
+ if not self.sametitle(pageitem['title'], title):
+ pywikibot.warning(
+ u"loadpageprops: Query on %s returned data on '%s'"
+ % (page, pageitem['title']))
+ continue
+ api.update_page(page, pageitem)
+
def loadimageinfo(self, page, history=False):
"""Load image info from api and save in page attributes
--
To view, visit https://gerrit.wikimedia.org/r/82769
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0752f1560e0e93cba0e01a9a1ea834e35f8a725c
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: Limit ItemPage preloading to existing pages
......................................................................
Limit ItemPage preloading to existing pages
[[d:Q6]] does not exist, so just go up to and including [[d:Q5]].
Change-Id: I712d0138529af20dd1c88c726aa1122ae3567b46
---
M tests/site_tests.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/tests/site_tests.py b/tests/site_tests.py
index 14720f8..6d41c98 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -202,7 +202,7 @@
datasite = mysite.data_repository()
- items = [pywikibot.ItemPage(datasite, 'q' + str(num)) for num in range(1, 11)]
+ items = [pywikibot.ItemPage(datasite, 'q' + str(num)) for num in range(1, 6)]
for page in datasite.preloaditempages(items):
self.assertTrue(hasattr(page, '_content'))
--
To view, visit https://gerrit.wikimedia.org/r/82891
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I712d0138529af20dd1c88c726aa1122ae3567b46
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: Implement a preloading generator for ItemPages
......................................................................
Implement a preloading generator for ItemPages
Since it becomes near impossible to find which specific
item lines up with which original page object, we just
create new objects and yield those.
WikibasePage.__defined_by had to be renamed so it is
no longer a private method.
Change-Id: I05917eb982f41dcdc92d8e45292ba6c27eac47f7
---
M pywikibot/page.py
M pywikibot/pagegenerators.py
M pywikibot/site.py
M tests/site_tests.py
4 files changed, 69 insertions(+), 2 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 7956ae1..1244a72 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -2293,7 +2293,11 @@
del self._link._title
return Page(self).title(**kwargs)
+ @deprecated("_defined_by")
def __defined_by(self, singular=False):
+ return self._defined_by(singular=singular)
+
+ def _defined_by(self, singular=False):
"""
returns the parameters needed by the API to identify an item.
Once an item's "p/q##" is looked up, that will be used for all future
@@ -2342,7 +2346,7 @@
args can be used to specify custom props.
"""
if force or not hasattr(self, '_content'):
- data = self.repo.loadcontent(self.__defined_by(), *args)
+ data = self.repo.loadcontent(self._defined_by(), *args)
self.id = data.keys()[0]
self._content = data[self.id]
if 'lastrevid' in self._content:
@@ -2429,7 +2433,7 @@
baserevid = self.lastrevid
else:
baserevid = None
- updates = self.repo.editEntity(self.__defined_by(singular=True), data,
+ updates = self.repo.editEntity(self._defined_by(singular=True), data,
baserevid=baserevid, **kwargs)
self.lastrevid = updates['entity']['lastrevid']
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index c6e8c7d..0fbfa63 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -797,16 +797,44 @@
site = page.site
sites.setdefault(site, []).append(page)
if len(sites[site]) >= step:
+ # if this site is at the step, process it
group = sites[site]
sites[site] = []
for i in site.preloadpages(group, step):
yield i
for site in sites:
if sites[site]:
+ # process any leftover sites that never reached the step
for i in site.preloadpages(sites[site], step):
yield i
+def PreloadingItemGenerator(generator, step=50):
+ """
+ Yield preloaded pages taken from another generator.
+
+ Function basically is copied from above, but for ItemPage's
+
+ @param generator: pages to iterate over
+ @param step: how many pages to preload at once
+ """
+ sites = {}
+ for page in generator:
+ site = page.site
+ sites.setdefault(site, []).append(page)
+ if len(sites[site]) >= step:
+ # if this site is at the step, process it
+ group = sites[site]
+ sites[site] = []
+ for i in site.preloaditempages(group, step):
+ yield i
+ for site in sites:
+ if sites[site]:
+ # process any leftover sites that never reached the step
+ for i in site.preloaditempages(sites[site], step):
+ yield i
+
+
def NewimagesPageGenerator(step=None, total=None, site=None):
if site is None:
site = pywikibot.Site()
diff --git a/pywikibot/site.py b/pywikibot/site.py
index ef00047..920b414 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -3398,6 +3398,32 @@
raise pywikibot.data.api.APIError, data['errors']
return data['entities']
+ def preloaditempages(self, pagelist, groupsize=50):
+ """Yields ItemPages with content prefilled.
+
+ Note that pages will be iterated in a different order
+ than in the underlying pagelist.
+
+ @param pagelist: an iterable that yields ItemPage objects
+ @param groupsize: how many pages to query at a time
+ @type groupsize: int
+ """
+ from pywikibot.tools import itergroup
+ for sublist in itergroup(pagelist, groupsize):
+ req = {'ids': [], 'titles': [], 'sites': []}
+ for p in sublist:
+ ident = p._defined_by()
+ for key in ident:
+ req[key].append(ident[key])
+
+ req = api.Request(site=self, action='wbgetentities', **req)
+ data = req.submit()
+ for qid in data['entities']:
+ item = pywikibot.ItemPage(self, qid)
+ item._content = data['entities'][qid]
+ item.get() # This parses the json and preloads the various properties
+ yield item
+
def getPropertyType(self, prop):
"""
This is used sepecifically because we can cache
diff --git a/tests/site_tests.py b/tests/site_tests.py
index cb29a0e..bf9e33f 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -202,6 +202,15 @@
if count >= 5:
break
+ def testItemPreload(self):
+ """Test that ItemPage preloading works"""
+
+ datasite = mysite.data_repository()
+
+ items = [pywikibot.ItemPage(datasite, 'q' + str(num)) for num in range(1, 11)]
+ for page in datasite.preloaditempages(items):
+ self.assertTrue(hasattr(page, '_content'))
+
def testLinkMethods(self):
"""Test site methods for getting links to and from a page"""
--
To view, visit https://gerrit.wikimedia.org/r/80789
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I05917eb982f41dcdc92d8e45292ba6c27eac47f7
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot