Pywikipedia-svn

pywikipedia-svn@lists.wikimedia.org

5163 discussions

SVN: [11382] trunk/pywikipedia/wikipedia.py
by drtrigon＠svn.wikimedia.org 18 Apr '13

18 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11382 Revision: 11382 Author: drtrigon Date: 2013-04-18 16:05:32 +0000 (Thu, 18 Apr 2013) Log Message: ----------- improvement; completed wikidata class structure bug fix; -simulation crashed in editclaim if no claims present Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-18 15:49:44 UTC (rev 11381) +++ trunk/pywikipedia/wikipedia.py 2013-04-18 16:05:32 UTC (rev 11382) @@ -4013,7 +4013,11 @@ getentities : Get the data for multiple Wikibase entities DEPRECATED: please use get() instead of getentities() searchentities : Search for entities - + linktitles : Associate articles on two different wikis with an item + IMPLEMENTATION: open - not done yet + removeclaim : Remove Wikibase claims + removereferences : Remove one or more references of the same statement + IMPLEMENTATION: open - not done yet """ def __init__(self, source, title=None, *args, **kwargs): if isinstance(source, basestring): @@ -4072,6 +4076,12 @@ elif items['type'] == u'sitelink': params['linksite'] = items['site'] + u'wiki' params['linktitle'] = items['title'] + elif items['type'] == u'label': + raise NotImplementedError( + u'Wikidata action type "%s" was not implemented yet' % items['type']) + elif items['type'] == u'aliases': + raise NotImplementedError( + u'Wikidata action type "%s" was not implemented yet' % items['type']) else: raise NotImplementedError( u'Wikidata action type "%s" is unknown' % items['type']) @@ -4298,7 +4308,7 @@ value = "{\"entity-type\":\"item\",\"numeric-id\":%s}" % value else: pass - claims = self.get()['claims'] + claims = self.get()['claims'] if 'claims' in self.get() else [] theclaim = None for claim in claims: if claim['m'][1] == propertyID: @@ -4683,6 +4693,14 @@ links[code]) for code in links] return self._interwiki + def linktitles(self): + raise NotImplementedError( + u'Wikidata member function "wblinktitles" was not implemented yet') + + def removereferences(self): + raise NotImplementedError( + u'Wikidata member function "wbremovereferences" was not implemented yet') + wikidataPage = DataPage #keep compatible

1 0

SVN: [11381] trunk/pywikipedia/wikipedia.py
by amir＠svn.wikimedia.org 18 Apr '13

18 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11381 Revision: 11381 Author: amir Date: 2013-04-18 15:49:44 +0000 (Thu, 18 Apr 2013) Log Message: ----------- adding removeclaim method for a DataPage Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-18 09:21:17 UTC (rev 11380) +++ trunk/pywikipedia/wikipedia.py 2013-04-18 15:49:44 UTC (rev 11381) @@ -4421,7 +4421,73 @@ if (u'warnings' in data) and \ (not data[u'warnings'][u'messages'][u'0'][u'name'] == u'edit-no-change'): output(str(data[u'warnings'])) - + def removeclaim(self,WDproperty ,value = None, raw_value=False, botflag= True, token=None, sysop=False): + if isinstance(WDproperty, int): + propertyID = WDproperty + elif isinstance(WDproperty, basestring): + try: + propertyID=int(WDproperty) + except ValueError: + try: + propertyID = int( + WDproperty.replace("p", "").replace("P", "")) + except ValueError: + search = self.searchentities(WDproperty, 'property', + lang=self._originSite.lang) + propertyID = int(search[0]["id"].replace("p", "")) + else: + pass + else: + pass + else: + raise RuntimeError("Unknown property type: %s" % WDproperty) + if not raw_value: + if isinstance(value, int) or not value: + pass + elif isinstance(value, unicode): # for 'string' entity-type + value = json.dumps(value) + elif isinstance(value, basestring): # for 'quantity' entity-type + try: + value=int(value) + except ValueError: + try: + value=int(value.replace("q","").replace("Q", "")) + except ValueError: + search=self.searchentities(value, 'item', + lang=self._originSite.lang) + value=int(search[0]["id"].replace("q", "")) + else: + pass + else: + pass + else: + raise RuntimeError("Unknown property type: %s" % value) + else: + pass + claims = self.get()['claims'] + theclaim = [] + if value: + for claim in claims: + if claim['m'][1] == propertyID and (int(claim['m'][3][u'numeric-id'])==value or (not value)): + theclaim.append(claim['g']) + if not theclaim: + raise RuntimeError("The claim %s hasn't been used in the item" % propertyID) + params = { + 'action': 'wbremoveclaims', + 'claim': "|".join(theclaim), + } + if token: + params['token'] = token + else: + params['token'] = self.site().getToken(sysop = sysop) + if botflag: + params['bot'] = 1 + output(u"Remving claim from %s" % self.title()) + data = query.GetData(params, self.site(), sysop=sysop) + if 'error' in data: + raise RuntimeError("API query error: %s" % data) + if 'warnings' in data: + output(str(data[u'warnings'])) def _getentity(self,force=False, get_redirect=False, throttle=True, sysop=False, change_edit_time=True): """Returns items of a entity in a dictionary

1 0

SVN: [11380]
by valhallasw＠svn.wikimedia.org 18 Apr '13

18 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11380 Revision: 11380 Author: valhallasw Date: 2013-04-18 09:21:17 +0000 (Thu, 18 Apr 2013) Log Message: ----------- API-based search: [] means /every/ namespace, not just 0 In the scrape-based search, the namespaces=[] parameter meant to search /every/ namespace, not just 0. This commit adapts the API search behavior to be consistent with this. Modified Paths: -------------- branches/rewrite/pywikibot/site.py trunk/pywikipedia/wikipedia.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2013-04-17 17:05:23 UTC (rev 11379) +++ branches/rewrite/pywikibot/site.py 2013-04-18 09:21:17 UTC (rev 11380) @@ -2250,7 +2250,7 @@ @param where: Where to search; value must be "text" or "titles" (many wikis do not support title search) @param namespaces: search only in these namespaces (defaults to 0) - @type namespaces: list of ints + @type namespaces: list of ints, or an empty list to signal all namespaces @param getredirects: if True, include redirects in results @param content: if True, load the current content of each iterated page (default False) @@ -2260,6 +2260,8 @@ raise Error("search: searchstring cannot be empty") if where not in ("text", "titles"): raise Error("search: unrecognized 'where' value: %s" % where) + if namespaces == []: + namespaces = [ns for ns in self.namespaces().keys() if ns >= 0] if not namespaces: pywikibot.warning(u"search: namespaces cannot be empty; using [0].") namespaces = [0] Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-17 17:05:23 UTC (rev 11379) +++ trunk/pywikipedia/wikipedia.py 2013-04-18 09:21:17 UTC (rev 11380) @@ -7018,6 +7018,8 @@ params['srlimit'] = number if namespaces: params['srnamespace'] = namespaces + if namespaces == []: + params['srnamespace'] = [ni for ni in (self.getNamespaceIndex(x) for x in self.namespaces()) if ni >= 0] offset = 0 while offset < number or not number:

1 0

SVN: [11379] trunk/pywikipedia/wikipedia.py
by drtrigon＠svn.wikimedia.org 17 Apr '13

17 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11379 Revision: 11379 Author: drtrigon Date: 2013-04-17 17:05:23 +0000 (Wed, 17 Apr 2013) Log Message: ----------- bug fix; enable continuation in searchentities Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-17 15:48:52 UTC (rev 11378) +++ trunk/pywikipedia/wikipedia.py 2013-04-17 17:05:23 UTC (rev 11379) @@ -4153,7 +4153,7 @@ return 302, response.msg, data['success'] return 302, response.msg, False - def createitem(self, summary=None, value=None, watchArticle=False, minorEdit=True, + def createitem(self, summary=u'', value=None, watchArticle=False, minorEdit=True, token=None, newToken=False, sysop=False, captcha=None, botflag=True, maxTries=-1): """Creating an item @@ -4571,8 +4571,14 @@ if entitytype: params['type']=entitytype # retrying is done by query.GetData - data = query.GetData(params, self.site(), sysop=sysop) - search = data['search'] + search = [] + while True: + data = query.GetData(params, self.site(), sysop=sysop) + search += data['search'] + if u'search-continue' in data: + params[u'continue'] = data[u'search-continue'] + else: + break if 'error' in data: raise RuntimeError("API query error: %s" % data)

1 0

SVN: [11378] trunk/pywikipedia/pywikibot/textlib.py
by amir＠svn.wikimedia.org 17 Apr '13

17 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11378 Revision: 11378 Author: amir Date: 2013-04-17 15:48:52 +0000 (Wed, 17 Apr 2013) Log Message: ----------- removing parenthesis as a not-at-the-end-mark per bug #3610818 Modified Paths: -------------- trunk/pywikipedia/pywikibot/textlib.py Modified: trunk/pywikipedia/pywikibot/textlib.py =================================================================== --- trunk/pywikipedia/pywikibot/textlib.py 2013-04-17 14:38:34 UTC (rev 11377) +++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-17 15:48:52 UTC (rev 11378) @@ -837,7 +837,7 @@ # Note: While allowing dots inside URLs, MediaWiki will regard # dots at the end of the URL as not part of that URL. # The same applies to comma, colon and some other characters. - notAtEnd = '\]\s\.:;,<>"\|\)' + notAtEnd = '\]\s\.:;,<>"\|' # So characters inside the URL can be anything except whitespace, # closing squared brackets, quotation marks, greater than and less # than, and the last character also can't be parenthesis or another

1 0

SVN: [11377] trunk/pywikipedia/sum_disc.py
by drtrigon＠svn.wikimedia.org 17 Apr '13

17 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11377 Revision: 11377 Author: drtrigon Date: 2013-04-17 14:38:34 +0000 (Wed, 17 Apr 2013) Log Message: ----------- bug fix; for cleanup with unicode data ('?\195?\164'), follow-up to r11214 (and all before that) Modified Paths: -------------- trunk/pywikipedia/sum_disc.py Modified: trunk/pywikipedia/sum_disc.py =================================================================== --- trunk/pywikipedia/sum_disc.py 2013-04-17 09:40:00 UTC (rev 11376) +++ trunk/pywikipedia/sum_disc.py 2013-04-17 14:38:34 UTC (rev 11377) @@ -36,10 +36,6 @@ will cause no problem for the bot, because the entries are also written to the history. -The following parameters are supported: - -&params; - All other parameters will be ignored. Syntax example: @@ -1116,7 +1112,7 @@ for line in text.splitlines(): try: #date = time.strptime(u'abc', u'; %d. %B %Y') - date = time.strptime(line, str(self._param['parse_msg'][u'start'])) + date = time.strptime(line.encode('utf-8'), str(self._param['parse_msg'][u'start'])) #date = time.strptime(str(line), str(self._param['parse_msg'][u'start'])) date = datetime.datetime.fromtimestamp(time.mktime(date)) diff = (today - date).days

1 0

SVN: [11376] trunk/pywikipedia/subster.py
by drtrigon＠svn.wikimedia.org 17 Apr '13

17 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11376 Revision: 11376 Author: drtrigon Date: 2013-04-17 09:40:00 +0000 (Wed, 17 Apr 2013) Log Message: ----------- wikidata code cleaned and finalized added future wikidata TODO hints Modified Paths: -------------- trunk/pywikipedia/subster.py Modified: trunk/pywikipedia/subster.py =================================================================== --- trunk/pywikipedia/subster.py 2013-04-16 20:58:15 UTC (rev 11375) +++ trunk/pywikipedia/subster.py 2013-04-17 09:40:00 UTC (rev 11376) @@ -83,6 +83,8 @@ from pywikibot.comms import http +# TODO: think about what config to move to 'subster-config.css' (per wiki) +# e.g. 'VerboseMessage', 'data_VerboseMessage', ... bot_config = { # unicode values 'BotName': pywikibot.config.usernames[pywikibot.config.family][pywikibot.config.mylang], @@ -531,7 +533,8 @@ @type substed_content: string """ # DRTRIGON-130: convert talk page result to wikidata(base) - #res, i = {}, 0 + # TODO: consider format; every line starting with "|" is data + # TODO: combine with 'outputContentDiff' in order to update changed only res = {} for line in substed_content.splitlines(): #data = self.get_var_regex('(.*?)', '(.*?)').findall(line) @@ -564,7 +567,7 @@ item = el[2] if item not in data: pywikibot.output(u'Value "%s" not found.' % (item,)) - continue + data[item] = u'%s: N/A' % self._bot_config['BotName'] if len(el) > 3: propid = el[3] @@ -580,9 +583,9 @@ (element[u'aliases'][0], dataoutpage.title(asLink=True), item, data[item])) dataoutpage.editclaim(u'p%s' % propid, data[item], refs={"p%s" % propid: - [{"snaktype": "value", - "property":"p%s" % propid, - "datavalue": {u'type': u'string', + [{"snaktype": "value", + "property": "p%s" % propid, + "datavalue": {u'type': u'string', u'value': datapage.title()}},]}, comment=summary)

1 0

SVN: [11375] trunk/pywikipedia/subster.py
by drtrigon＠svn.wikimedia.org 16 Apr '13

16 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11375 Revision: 11375 Author: drtrigon Date: 2013-04-16 20:58:15 +0000 (Tue, 16 Apr 2013) Log Message: ----------- improvement; added references support for wikidata (follow-up to r11374) Modified Paths: -------------- trunk/pywikipedia/subster.py Modified: trunk/pywikipedia/subster.py =================================================================== --- trunk/pywikipedia/subster.py 2013-04-16 20:48:25 UTC (rev 11374) +++ trunk/pywikipedia/subster.py 2013-04-16 20:58:15 UTC (rev 11375) @@ -574,13 +574,17 @@ summary = u'Bot: update data because of configuration on %s.' % page.title(asLink=True) buf = dataoutpage.get() claim = [ claim for claim in buf[u'claims'] if (claim['m'][1] == propid) ] + # TODO: does this check (if) work with multiple claims per property? if (not claim) or (claim[0]['m'][3] != data[item]): pywikibot.output(u'%s in %s <--- %s = %s' %\ (element[u'aliases'][0], dataoutpage.title(asLink=True), item, data[item])) - dataoutpage.editclaim(u'p%s' % propid, data[item], -# refs={(self._bot_config['data_PropertyId'], datapage.title()),}, - comment=summary) + refs={"p%s" % propid: + [{"snaktype": "value", + "property":"p%s" % propid, + "datavalue": {u'type': u'string', + u'value': datapage.title()}},]}, + comment=summary) def get_var_regex(self, var, cont='.*?'): """Get regex used/needed to find the tags to replace.

1 0

SVN: [11374] trunk/pywikipedia/wikipedia.py
by drtrigon＠svn.wikimedia.org 16 Apr '13

16 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11374 Revision: 11374 Author: drtrigon Date: 2013-04-16 20:48:25 +0000 (Tue, 16 Apr 2013) Log Message: ----------- improvement; allow generic or "raw" references in editclaim Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-16 18:54:08 UTC (rev 11373) +++ trunk/pywikipedia/wikipedia.py 2013-04-16 20:48:25 UTC (rev 11374) @@ -4313,11 +4313,12 @@ params['token'] = token or self.site().getToken(sysop=sysop) if botflag: params['bot'] = 1 - output(u"Changing %s" % self.title()) + output(u"Changing claim in item %s" % self.title()) data = query.GetData(params, self.site(), sysop=sysop) if 'error' in data: raise RuntimeError("API query error: %s" % data) - if u'warnings' in data: + if (u'warnings' in data) and \ + (not data[u'warnings'][u'messages'][u'0'][u'name'] == u'edit-no-change'): output(str(data[u'warnings'])) guid=theclaim['g'] else: @@ -4331,7 +4332,7 @@ params['token'] = token or self.site().getToken(sysop=sysop) if botflag: params['bot'] = 1 - output(u"Creating %s" % self.title()) + output(u"Creating claim in item %s" % self.title()) data = query.GetData(params, self.site(), sysop=sysop) if 'error' in data: raise RuntimeError("API query error: %s" % data) @@ -4340,49 +4341,65 @@ guid=data['claim']['id'] if refs: snak = [] - for ref in refs: - if isinstance(ref,basestring): - raise RuntimeError( - "the references must be like this: {(ref1, value1), (ref2, value2)}") - for i in range(2): - if isinstance(ref[i], int): - value = ref[i] - elif isinstance(ref[i], basestring): - try: - value = int(ref[i]) - except ValueError: + if isinstance(refs, dict): + # the references must be like this: + # {"p?": [{"snaktype": "value", + # "property":"p?", + # "datavalue": {u'type': u'string', u'value': u'...'}}, + # {"snaktype": "value", ... }}, ]} + pass + elif isinstance(refs, set): + # the references must be like this: + # {(ref1, value1), (ref2, value2)} + for ref in refs: + if isinstance(ref, basestring): + raise RuntimeError( + "the references must be like this: {(ref1, value1), (ref2, value2)}") + for i in range(2): + if isinstance(ref[i], int): + value = ref[i] + elif isinstance(ref[i], basestring): try: - value = int( - ref[i].lower().replace("Q", - "").replace("P", "")) + value = int(ref[i]) except ValueError: - if i == 0: - typesearch = 'property' + try: + value = int( + ref[i].lower().replace("Q", + "").replace("P", "")) + except ValueError: + if i == 0: + typesearch = 'property' + else: + typesearch = 'item' + search=self.searchentities( + ref[i], typesearch, + lang=self._originSite.lang) + value = int( + search[0]["id"].replace("q", + "").replace("p", + "")) else: - typesearch = 'item' - search=self.searchentities( - ref[i], typesearch, - lang=self._originSite.lang) - value = int( - search[0]["id"].replace("q", - "").replace("p", - "")) + pass else: pass else: - pass - else: - raise RuntimeError("Unknown item: %s" % ref[i]) - snak.append(value) - finalsnak = {} - for i in range(0, len(snak) / 2): - snaki = [ - {"snaktype": "value", - "property":"p"+str(snak[i*2]), - "datavalue": {"type": "wikibase-entityid", - "value": {"entity-type": "item", - "numeric-id": snak[(i * 2) + 1]}}}] - finalsnak["p%d" % snak[i * 2]] = snaki + raise RuntimeError("Unknown item: %s" % ref[i]) + snak.append(value) + else: + raise RuntimeError( + "the references format cannot be understood!") + if snak: + finalsnak = {} + for i in range(0, len(snak) / 2): + snaki = [ + {"snaktype": "value", + "property":"p"+str(snak[i*2]), + "datavalue": {"type": "wikibase-entityid", + "value": {"entity-type": "item", + "numeric-id": snak[(i * 2) + 1]}}}] + finalsnak["p%d" % snak[i * 2]] = snaki + else: + finalsnak=refs finalsnak=json.dumps(finalsnak) finalsnak=finalsnak.replace("'", '"') params = { @@ -4397,11 +4414,12 @@ params['token'] = self.site().getToken(sysop = sysop) if botflag: params['bot'] = 1 - output(u"Adding references to %s" % self.title()) + output(u"Adding references to claim in %s" % self.title()) data = query.GetData(params, self.site(), sysop=sysop) if 'error' in data: raise RuntimeError("API query error: %s" % data) - if 'warnings' in data: + if (u'warnings' in data) and \ + (not data[u'warnings'][u'messages'][u'0'][u'name'] == u'edit-no-change'): output(str(data[u'warnings'])) def _getentity(self,force=False, get_redirect=False, throttle=True,

1 0

SVN: [11373] branches/rewrite/scripts/claimit.py
by multichill＠svn.wikimedia.org 16 Apr '13

16 Apr '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11373 Revision: 11373 Author: multichill Date: 2013-04-16 18:54:08 +0000 (Tue, 16 Apr 2013) Log Message: ----------- Added crude dupe checking so we're not causing a big mess. Modified Paths: -------------- branches/rewrite/scripts/claimit.py Modified: branches/rewrite/scripts/claimit.py =================================================================== --- branches/rewrite/scripts/claimit.py 2013-04-16 18:22:45 UTC (rev 11372) +++ branches/rewrite/scripts/claimit.py 2013-04-16 18:54:08 UTC (rev 11373) @@ -30,10 +30,14 @@ return False for claim in claims: - pywikibot.output('Adding %s --> %s' % (claim.getID(), claim.getTarget().getID())) - item.addClaim(claim) - #TODO FIXME: We should add a source for each claim that is added - #TODO FIXME: We need to check that we aren't adding a duplicate + if claim.getID() in item.get().get('claims'): + pywikibot.output(u'A claim for %s already exists. Skipping' % (claim.getID(),)) + #TODO FIXME: This is a very crude way of dupe checking + else: + pywikibot.output('Adding %s --> %s' % (claim.getID(), claim.getTarget().getID())) + item.addClaim(claim) + #TODO FIXME: We should add a source for each claim that is added + #TODO FIXME: We need to check that we aren't adding a duplicate def main(): @@ -52,7 +56,6 @@ claim.setTarget(pywikibot.ItemPage(repo, commandline_claims[i+1])) claims.append(claim) - generator = gen.getCombinedGenerator() if generator:

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn