pywikibot

pywikibot@lists.wikimedia.org

3 participants
6830 discussions

[Pywikipedia-l] SVN: [4092] trunk/pywikipedia
by cosoleto＠svn.wikimedia.org 23 Aug '07

23 Aug '07

Revision: 4092 Author: cosoleto Date: 2007-08-23 08:30:49 +0000 (Thu, 23 Aug 2007) Log Message: ----------- economize_query() to skip sequence of numbers or wide comma separated lists. Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2007-08-22 21:52:43 UTC (rev 4091) +++ trunk/pywikipedia/config.py 2007-08-23 08:30:49 UTC (rev 4092) @@ -344,6 +344,14 @@ # Append length of URL to script result copyright_show_length = True +# By default the script try to identify and skip text that contents a wide +# comma separated list or only numbers. But sometimes that might be the +# only part unmodified of a slightly edited and not otherwise reported +# copyright violation. You can disable this feature to try to increase +# accuracy. + +copyright_economize_query = True + ############## FURTHER SETTINGS ############## # The bot can make some additional changes to each page it edits, e.g. fix Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 21:52:43 UTC (rev 4091) +++ trunk/pywikipedia/copyright.py 2007-08-23 08:30:49 UTC (rev 4092) @@ -75,10 +75,15 @@ __version__='$Id$' -# Try to skip quoted text +# Try to skip quoted text. exclude_quote = True -# No checks if the page is a disambiguation page +# If ratio between query length and number of commas is greater or equal +# to 'comma_ratio' then the script identify a comma separated list and +# don't send data to search engine. +comma_ratio = 5 + +# No checks if the page is a disambiguation page. skip_disambig = True appdir = "copyright/" @@ -323,6 +328,27 @@ f.close() # +# Ignore text that contents comma separated list, only numbers, +# punctuation... + +def economize_query(text) + # Comma separated list + if text.count(', ') > 4: + l = len(text) + c = text.count(', ') + r = 100 * c / l + + if r >= comma_ratio + return True + + # write_log("%d/%d/%d: %s\n" % (l,c,r,text), "copyright/skip" + str(r) + ".txt") + + # Numbers + if re.search('[^0-9\'*/,. +?:;-]{5}', text): + return False + return True + +# # Set regex used in cleanwikicode() to remove [[Image:]] tags # and regex used in check_in_source() to reject pages with # 'Wikipedia'. @@ -442,6 +468,11 @@ line = cleanwikicode(line) for search_words in mysplit(line, 31, " "): if len(search_words) > 120: + if config.copyright_economize_query: + if economize_query(search_words): + wikipedia.output('SKIP TEXT: ' + search_words) + consecutive = False + continue n_query += 1 #wikipedia.output(search_words) if config.copyright_max_query_for_page and n_query > config.copyright_max_query_for_page:

1 0

[Pywikipedia-l] SVN: [4091] trunk/pywikipedia
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4091 Author: cosoleto Date: 2007-08-22 21:52:43 +0000 (Wed, 22 Aug 2007) Log Message: ----------- Patch to disable search engine/sleep/stop/ignore if search engine refuse your query because you exceed daily quota. Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2007-08-22 20:52:26 UTC (rev 4090) +++ trunk/pywikipedia/config.py 2007-08-22 21:52:43 UTC (rev 4091) @@ -288,10 +288,10 @@ ############## SEARCH ENGINE SETTINGS ############## -# Some scripts allow querying Google via the Google Web API. To use this feature, you must -# install the pyGoogle module from http://pygoogle.sf.net/ and have a Google -# Web API license key. Note that -# Google doesn't give out license keys anymore. +# Some scripts allow querying Google via the Google Web API. To use this feature, +# you must install the pyGoogle module from http://pygoogle.sf.net/ and have a +# Google Web API license key. Note that Google doesn't give out license keys +# anymore. google_key = '' # Some scripts allow using the Yahoo! Search Web Services. To use this feature, @@ -326,6 +326,18 @@ # Number of attempts on connection error. copyright_connection_tries = 10 +# Behavior if an exceeded error occur. +# +# Possibilities: +# +# 0 = None +# 1 = Disable search engine +# 2 = Sleep (default) +# 3 = Stop + +copyright_exceeded_in_queries = 2 +copyright_exceeded_in_queries_sleep_hours = 6 + # Append last modified date of URL to script result copyright_show_date = True Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 20:52:26 UTC (rev 4090) +++ trunk/pywikipedia/copyright.py 2007-08-22 21:52:43 UTC (rev 4091) @@ -659,6 +659,20 @@ url.append((add_item, engine, comment)) return +def exceeded_in_queries(engine): + """Behavior if an exceeded error occur.""" + + # Disable search engine + if config.copyright_exceeded_in_queries == 1: + exec('config.copyright_' + engine + ' = False') + # Sleeping + if config.copyright_exceeded_in_queries == 2: + print "Got a queries exceeded error. Sleeping for %d hours..." % (config.copyright_exceeded_in_queries_sleep_hours) + time.sleep(config.copyright_exceeded_in_queries_sleep_hours * 60 * 60) + # Stop execution + if config.copyright_exceeded_in_queries == 3: + raise 'Got a queries exceeded error.' + def get_results(query, numresults = 10): url = list() query = re.sub("[()\"<>]", "", query) @@ -677,9 +691,13 @@ except KeyboardInterrupt: raise except Exception, err: - #SOAP.faultType: <Fault SOAP-ENV:Server: Exception from service object: - # Daily limit of 1000 queries exceeded for key xxx> print "Got an error ->", err + # + # SOAP.faultType: <Fault SOAP-ENV:Server: Exception from service object: + # Daily limit of 1000 queries exceeded for key ***> + # + if 'Daily limit' in str(err): + exceeded_in_queries('google') if search_request_retry: search_request_retry -= 1 if config.copyright_yahoo: @@ -696,6 +714,8 @@ search_request_retry = 0 except Exception, err: print "Got an error ->", err + if 'limit exceeded' in str(err): + exceeded_in_queries('yahoo') if search_request_retry: search_request_retry -= 1 if config.copyright_msn:

1 0

[Pywikipedia-l] SVN: [4090] trunk/pywikipedia
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4090 Author: cosoleto Date: 2007-08-22 20:52:26 +0000 (Wed, 22 Aug 2007) Log Message: ----------- Added support for Live Search Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2007-08-22 20:35:48 UTC (rev 4089) +++ trunk/pywikipedia/config.py 2007-08-22 20:52:26 UTC (rev 4090) @@ -288,13 +288,10 @@ ############## SEARCH ENGINE SETTINGS ############## -# Some scripts allow querying Google either via the Google Web API, or by -# just parsing the HTML from the Google website. -# To use the Google Web API, you must install the pyGoogle module from -# http://pygoogle.sf.net/ and have a Google Web API license key. Note that +# Some scripts allow querying Google via the Google Web API. To use this feature, you must +# install the pyGoogle module from http://pygoogle.sf.net/ and have a Google +# Web API license key. Note that # Google doesn't give out license keys anymore. -# If you don't enter a google license key in your user config file, the scripts -# will just parse the raw HTML code from the website. google_key = '' # Some scripts allow using the Yahoo! Search Web Services. To use this feature, @@ -302,17 +299,23 @@ # and get a Yahoo AppID from http://developer.yahoo.com yahoo_appid = '' +# To use Windows Live Search web service you must get an AppID from +# http://search.msn.com/developer +msn_appid = '' + ############## COPYRIGHT SETTINGS ############## # Enable/disable search engine in copyright.py script copyright_google = True copyright_yahoo = True +copyright_msn = False # Perform a deep check, loading URLs to search if 'Wikipedia' is present. # This may be useful to improve number of correct results. If you haven't # a fast connection, you might want to keep they disabled. copyright_check_in_source_google = False copyright_check_in_source_yahoo = False +copyright_check_in_source_msn = False # Limit number of queries for page. copyright_max_query_for_page = 25 Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 20:35:48 UTC (rev 4089) +++ trunk/pywikipedia/copyright.py 2007-08-22 20:52:26 UTC (rev 4090) @@ -596,7 +596,8 @@ def add_in_urllist(url, add_item, engine): if (engine == 'google' and config.copyright_check_in_source_google) or \ - (engine == 'yahoo' and config.copyright_check_in_source_yahoo): + (engine == 'yahoo' and config.copyright_check_in_source_yahoo) or \ + (engine == 'msn' and config.copyright_check_in_source_msn): check_in_source = True else: check_in_source = False @@ -697,32 +698,40 @@ print "Got an error ->", err if search_request_retry: search_request_retry -= 1 - #if search_in_msn: - # ## max_query_len = 150? - # from __SOAPpy import WSDL - # print " msn query..." - # wsdl_url = 'http://soap.search.msn.com/webservices.asmx?wsdl' - # server = WSDL.Proxy(wsdl_url) - # params = {'AppID': config.msn_appid, 'Query': '-Wikipedia "' + query + '"', 'CultureInfo': 'en-US', 'SafeSearch': 'Off', 'Requests': { - # 'SourceRequest':{'Source': 'Web', 'Offset': 0, 'Count': 10, 'ResultFields': 'All',}}} - # - # search_request_retry = config.copyright_connection_tries - # results = '' - # while search_request_retry: - # try: - # server_results = server.Search(Request = params) - # search_request_retry = 0 - # if server_results.Responses[0].Results: - # results = server_results.Responses[0].Results[0] - # except Exception, err: - # print "Got an error ->", err - # search_request_retry -= 1 - # for entry in results: - # try: - # add_in_urllist(url, entry.Url, 'msn') - # except AttributeError: - # print "attrib ERROR" + if config.copyright_msn: + #max_query_len = 150? + from SOAPpy import WSDL + print " Live query..." + try: + server = WSDL.Proxy('http://soap.search.msn.com/webservices.asmx?wsdl') + except: + print "Live Search Error" + raise + params = {'AppID': config.msn_appid, 'Query': '-Wikipedia "' + query + '"', 'CultureInfo': 'en-US', 'SafeSearch': 'Off', 'Requests': { + 'SourceRequest':{'Source': 'Web', 'Offset': 0, 'Count': 10, 'ResultFields': 'All',}}} + + search_request_retry = config.copyright_connection_tries + results = '' + while search_request_retry: + try: + server_results = server.Search(Request = params) + search_request_retry = 0 + if server_results.Responses[0].Results: + results = server_results.Responses[0].Results[0] + except Exception, err: + print "Got an error ->", err + if search_request_retry: + search_request_retry -= 1 + + if results: + # list or instance? + if type(results) == type([]): + for entry in results: + add_in_urllist(url, entry.Url, 'msn') + else: + add_in_urllist(url, results.Url, 'msn') + offset = 0 for i in range(len(url)): if check_list(url[i + offset][0], excl_list, verbose = True):

1 0

[Pywikipedia-l] SVN: [4089] trunk/pywikipedia
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4089 Author: cosoleto Date: 2007-08-22 20:35:48 +0000 (Wed, 22 Aug 2007) Log Message: ----------- Add last modified date and length of URL to copyright.py's result. The script can add also a Google cache link if a 400 HTTP status code is found. check_in_source() stuff rewrote object-oriented and equiped with a simple code to decode UTF-8 data. Modified Paths: -------------- trunk/pywikipedia/config.py trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2007-08-22 18:03:28 UTC (rev 4088) +++ trunk/pywikipedia/config.py 2007-08-22 20:35:48 UTC (rev 4089) @@ -323,6 +323,12 @@ # Number of attempts on connection error. copyright_connection_tries = 10 +# Append last modified date of URL to script result +copyright_show_date = True + +# Append length of URL to script result +copyright_show_length = True + ############## FURTHER SETTINGS ############## # The bot can make some additional changes to each page it edits, e.g. fix Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 18:03:28 UTC (rev 4088) +++ trunk/pywikipedia/copyright.py 2007-08-22 20:35:48 UTC (rev 4089) @@ -456,8 +456,11 @@ search_words = search_words[:search_words.rindex(" ")] results = get_results(search_words) group_url = '' - for url, engine in results: - group_url += '\n*%s - %s' % (engine, url) + for url, engine, comment in results: + if comment: + group_url += '\n*%s - %s (%s)' % (engine, url, "; ".join(comment)) + else: + group_url += '\n*%s - %s' % (engine, url) if results: group_url_list = group_url.splitlines() group_url_list.sort() @@ -482,51 +485,177 @@ source_seen = set() positive_source_seen = set() -def check_in_source(url): +class NoWebPage(Exception): + """Web page does not exist (404)""" + +class URL_exclusion(Exception): + """URL in exclusion list""" + +class WebPage(object): """ - Sources may be different from search engine database and include mentions of - Wikipedia. This function avoid also errors in search results that can occurs - either with Google and Yahoo! service. """ - import urllib2 - global excl_list, source_seen, positive_source_seen - if url in positive_source_seen: - return True + def __init__(self, url): + """ + """ - if url in source_seen: - return False + if check_list(url, excl_list): + raise URL_exclusion - if check_list(url, excl_list): - return False + self._url = url - # very experimental code - if not url[-4:] in [".pdf", ".doc", ".ppt"]: try: - resp = urllib2.urlopen(url) - text = resp.read() - #resp.close() - except urllib2.HTTPError: + self._urldata = urllib2.urlopen(urllib2.Request(self._url, None, { 'User-Agent': wikipedia.useragent })) + #except httplib.BadStatusLine, line: + # print 'URL: %s\nBad status line: %s' % (url, line) + except urllib2.HTTPError, err: + print "HTTP error: %d / %s (%s)" % (err.code, err.msg, url) + #if err.code == 404: + if err.code >= 400: + raise NoWebPage + return None + #except urllib2.URLError: + except Exception, err: + print "ERROR: %s" % (err) + + self._lastmodified = self._urldata.info().getdate('Last-Modified') + self._length = self._urldata.info().getheader('Content-Length') + self._content_type = self._urldata.info().getheader('Content-Type') + + def length(self): + if hasattr(self, '_length'): + if self._length: + return int(self._length) + if hasattr(self, '_contents'): + return len(self._contents) + + # print "No length for " + self._url + + return None + + def lastmodified(self): + if hasattr(self, '_lastmodified'): + return self._lastmodified + return None + + def get(self, force = False): + """ + """ + + # Exclude URL with listed file extension. + if self._url[-4:] in [".pdf", ".doc", ".ppt"]: + raise URL_exclusion + + # Make sure we did try to get the contents once + if not hasattr(self, '_contents'): + self._contents = self._urldata.read() + return self._contents + return None + + def check_in_source(self): + """ + Sources may be different from search engine database and include mentions of + Wikipedia. This function avoid also errors in search results that can occurs + either with Google and Yahoo! service. + """ + global excl_list, source_seen, positive_source_seen + + if not hasattr(self, '_urldata'): return False - if reWikipediaC.search(text): - # if 'wikipedia' in text.lower(): - excl_list += [url] - #write_log(url + '\n', "copyright/sites_with_'wikipedia'.txt") - positive_source_seen.add(url) + if self._url in positive_source_seen: return True - else: - #write_log(url + '\n', "copyright/sites_without_'wikipedia'.txt") - source_seen.add(url) - return False + if self._url in source_seen: + return False + + text = self.get() + + # Character encoding conversion if 'Content-Type' field has + # charset attribute set to UTF-8. + + if text: + if 'utf-8' in self._content_type.lower(): + text = text.decode("utf-8", 'replace') + else: + # <META> declaration with "http-equiv" set to "Content-Type" in HTML document. + if 'text/html' in self._content_type and (re.search("(?is)<meta\s.*?charset\s*=\s*[\"\']*\s*UTF-8.*?>", text) or re.search("(?is)<\?.*?encoding\s*=\s*[\"\']*\s*UTF-8.*?\?>", text)): + text = text.decode("utf-8", 'replace') + + m = reWikipediaC.search(text) + if m: + excl_list += [self._url] + write_log("%s (%s)\n" % (self._url, m.group()), "copyright/sites_with_'wikipedia'.txt") + positive_source_seen.add(self._url) + return True + else: + write_log(self._url + '\n', "copyright/sites_without_'wikipedia'.txt") + source_seen.add(self._url) + return False + def add_in_urllist(url, add_item, engine): + + if (engine == 'google' and config.copyright_check_in_source_google) or \ + (engine == 'yahoo' and config.copyright_check_in_source_yahoo): + check_in_source = True + else: + check_in_source = False + + if check_in_source or config.copyright_show_date or config.copyright_show_length: + s = None + cache = False + + # list to store date, length, cache URL + comment = list() + + try: + s = WebPage(add_item) + except URL_exclusion: + pass + except NoWebPage: + cache = True + + if s: + # Before of add url in result list, perform the check in source + if check_in_source: + if s.check_in_source(): + return + + if config.copyright_show_date: + date = s.lastmodified() + if date: + if date[:3] != time.localtime()[:3]: + comment.append("%s/%s/%s" % (date[2], date[1], date[0])) + + unit = 'bytes' + + if config.copyright_show_length: + length = s.length() + if length: + # convert in kilobyte + length /= 1024 + unit = 'KB' + if length > 1024: + # convert in megabyte + length /= 1024 + unit = 'MB' + if length > 0: + comment.append("%d %s" % (length, unit)) + + if cache: + if engine == 'google': + comment.append('[http://www.google.com/search?sourceid=navclient&q=cache:%s google cache]' % add_item[7:]) + elif engine == 'yahoo': + cache = False + elif engine == 'msn': + cache = False + for i in range(len(url)): if add_item in url[i]: if engine not in url[i][1]: - url[i] = (add_item, url[i][1] + ', ' + engine) + url[i] = (add_item, url[i][1] + ', ' + engine, comment) return - url.append((add_item, engine)) + url.append((add_item, engine, comment)) return def get_results(query, numresults = 10): @@ -543,9 +672,6 @@ data = google.doGoogleSearch('-Wikipedia "' + query + '"') search_request_retry = 0 for entry in data.results: - if config.copyright_check_in_source_google: - if check_in_source(entry.URL): - continue add_in_urllist(url, entry.URL, 'google') except KeyboardInterrupt: raise @@ -565,9 +691,6 @@ while search_request_retry: try: for entry in data.parse_results(): - if config.copyright_check_in_source_yahoo: - if check_in_source(entry.Url): - continue add_in_urllist(url, entry.Url, 'yahoo') search_request_retry = 0 except Exception, err:

1 0

[Pywikipedia-l] SVN: [4088] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4088 Author: cosoleto Date: 2007-08-22 18:03:28 +0000 (Wed, 22 Aug 2007) Log Message: ----------- Skip disambiguation page, minor fixes. Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 14:40:30 UTC (rev 4087) +++ trunk/pywikipedia/copyright.py 2007-08-22 18:03:28 UTC (rev 4088) @@ -20,14 +20,18 @@ -ng - Do not use Google -y - Use Yahoo! search engine -ny - Do not use Yahoo! +-l - Use Windows Live Search engine +-nl - Do not use Windows Live Search -maxquery - Stop after a specified number of queries for page (default: 25) -skipquery - Skip a number specified of queries --new - -output - Append results to a specified file (default: 'copyright/output.txt') + -file - Work on all pages given in a local text file. Will read any [[wiki link]] and use these articles. Argument can also be given as "-file:filename". +-new - Work on the 60 newest pages. If given as -new:x, will work + on the x newest pages. -cat - Work on all pages which are in a specific category. Argument can also be given as "-cat:categoryname". -subcat - When the pages to work on have been chosen by -cat, pages in @@ -66,13 +70,17 @@ # from __future__ import generators -import sys, re, codecs, os, time +import sys, re, codecs, os, time, urllib2, httplib import wikipedia, pagegenerators, catlib, config __version__='$Id$' +# Try to skip quoted text exclude_quote = True +# No checks if the page is a disambiguation page +skip_disambig = True + appdir = "copyright/" output_file = appdir + "output.txt" @@ -265,7 +273,7 @@ for page, path in exclusion_file_list(): if 'exclusion_list.txt' in path: - result_list += re.sub("</?pre>","", read_file(path, cut_comment = True)).splitlines() + result_list += re.sub("</?pre>","", read_file(path, cut_comment = True, cut_newlines = True)).splitlines() else: data = read_file(path) # wikipedia:en:Wikipedia:Mirrors and forks @@ -291,11 +299,12 @@ else: result_list += [re.sub(" .*", "", entry)] - result_list += read_file(appdir + 'exclusion_list.txt', cut_comment = True).splitlines() + result_list += read_file(appdir + 'exclusion_list.txt', cut_comment = True, cut_newlines = True).splitlines() return result_list -def read_file(filename, cut_comment = False): +def read_file(filename, cut_comment = False, cut_newlines = False): text = u"" + f = codecs.open(filename, 'r','utf-8') text = f.read() f.close() @@ -303,6 +312,9 @@ if cut_comment: text = re.sub(" ?#.*", "", text) + if cut_newlines: + text = re.sub("(?m)^\r?\n", "", text) + return text def write_log(text, filename = output_file): @@ -422,7 +434,6 @@ # Google limit queries to 32 words. - output = u"" n_query = 0 previous_group_url = 'none' @@ -450,13 +461,11 @@ if results: group_url_list = group_url.splitlines() group_url_list.sort() - group_url = '\n'.join(group_url_list) if previous_group_url == group_url: if consecutive: output += ' ' + search_words else: - output += '\n**' + search_words else: output += group_url + '\n**' + search_words @@ -632,6 +641,11 @@ except wikipedia.IsRedirectPage: original_text = page.get(get_redirect=True) + if skip_disambig: + if page.isDisambig(): + wikipedia.output(u'Page %s is a disambiguation page' % page.title()) + continue + # colors = [13] * len(page.title()) wikipedia.output(page.title()) @@ -648,8 +662,11 @@ return False return var +def setSavepath(path): + global output_file + output_file = path + def main(): - global output_file gen = None # pages which will be processed when the -page parameter is used PageTitles = [] @@ -682,13 +699,17 @@ config.copyright_yahoo = True elif arg == '-g': config.copyright_google = True + elif arg == '-l': + config.copyright_msn = True elif arg == '-ny': config.copyright_yahoo = False elif arg == '-ng': config.copyright_google = False + elif arg == '-nl': + config.copyright_msn = False elif arg.startswith('-output'): if len(arg) >= 8: - output_file = arg[8:] + setSavepath(arg[8:]) elif arg.startswith('-maxquery'): if len(arg) >= 10: config.copyright_max_query_for_page = int(arg[10:])

1 0

[Pywikipedia-l] SVN: [4087] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4087 Author: cosoleto Date: 2007-08-22 14:40:30 +0000 (Wed, 22 Aug 2007) Log Message: ----------- code cleanup, improvements in cleanwikicode() Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-22 08:50:02 UTC (rev 4086) +++ trunk/pywikipedia/copyright.py 2007-08-22 14:40:30 UTC (rev 4087) @@ -71,7 +71,6 @@ __version__='$Id$' -#search_in_msn = False exclude_quote = True appdir = "copyright/" @@ -250,12 +249,12 @@ print 'Getting page failed' return -def check_list(text, cl, debug=False): +def check_list(text, cl, verbose = False): for entry in cl: if entry: if text.find(entry) != -1: #print entry - if debug: + if verbose: print 'SKIP URL ' + text return True @@ -299,9 +298,11 @@ text = u"" f = codecs.open(filename, 'r','utf-8') text = f.read() + f.close() + if cut_comment: text = re.sub(" ?#.*", "", text) - f.close() + return text def write_log(text, filename = output_file): @@ -329,10 +330,10 @@ def cleanwikicode(text): if not text: return "" - #write_log(text+'\n', "debug_cleanwikicode1.txt") - text = re.sub('(?i)<p.*?>' ,'', text) - text = re.sub('(?i)</?div.*?>' ,'', text) - text = re.sub("(?i)</*small>", "", text) + + #write_log(text+'\n', "copyright/debug_cleanwikicode1.txt") + + text = re.sub('(?i)</?(p|u|i|b|em|div|span|font|small|big|code|tt).*?>', '', text) text = re.sub('(?i)<(/\s*)?br(\s*/)?>', '', text) text = re.sub('', '', text) text = re.sub('<', '<', text) @@ -345,34 +346,38 @@ text = re.sub('^[:*]?\s*[«][^»]+[»]\.?\s*(($|<ref>).*?($|</ref>))?\.?$', "", text) text = re.sub('^[:*]?\s*[“][^”]+[”]\.?\s*(($|<ref>).*?($|</ref>))?\.?$', "", text) - # exclude <ref> notes - text = re.sub ("<ref.*?>.*?</ref>", "", text) - # exclude wikitable - text = re.sub('(?m)^[ \t]*({\||[|!]).*', "", text) # remove URL text = re.sub('https?://[\w/.,;:@&=%#\\\?_!~*\'|()\"+-]+', ' ', text) + # remove Image tags text = reImageC.sub("", text) + # replace piped wikilink text = re.sub("\[\[[^\]]*?\|(.*?)\]\]", "\\1", text) + # remove unicode and polytonic template text = re.sub("(?i){{(unicode|polytonic)\|(.*?)}}", "\\1", text) - # remove <nowiki> tags - text = re.sub("</*nowiki>", "", text) - # remove template - text = re.sub('{{.*?}}', '', text) - # remove LaTeX staff - text = re.sub('<math>.*?</math>', '', text) - #text = text.replace("''", "") - text = text.replace("[", "") - text = text.replace("]", "") - text = re.sub('^[*:;]', '', text) - text = text.replace("", "") + text = re.sub(""" + (?xim) + ( + <ref.*?>.*?</ref> | # exclude <ref> notes + ^[\ \t]*({\||[|!]).* | # exclude wikitable + </*nowiki> | # remove <nowiki> tags + {{.*?}} | # remove template + <math>.*?</math> | # remove LaTeX staff + [\[\]] | # remove [, ] + ^[*:;]+ | # remove *, :, ; in begin of line +  | + ) + """, "", text) + # remove useless spaces + text = re.sub("(?m)(^[ \t]+|[ \t]+$)", "", text) + #if text: - # write_log(text+'\n', "debug_cleanwikicode2.txt") + # write_log(text+'\n', "copyright/debug_cleanwikicode2.txt") return text excl_list = exclusion_list() @@ -522,7 +527,7 @@ if config.copyright_google: import google google.LICENSE_KEY = config.google_key - print " google query..." + print " Google query..." search_request_retry = config.copyright_connection_tries while search_request_retry: try: @@ -543,7 +548,7 @@ search_request_retry -= 1 if config.copyright_yahoo: import yahoo.search.web - print " yahoo query..." + print " Yahoo query..." data = yahoo.search.web.WebSearch(config.yahoo_appid, query='"' + query.encode('utf_8') + '" -Wikipedia', results=numresults) @@ -588,7 +593,7 @@ offset = 0 for i in range(len(url)): - if check_list(url[i + offset][0], excl_list, debug = True): + if check_list(url[i + offset][0], excl_list, verbose = True): url.pop(i + offset) offset += -1 return url

1 0

[Pywikipedia-l] SVN: [4086] trunk/pywikipedia/weblinkchecker.py
by btongminh＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4086 Author: btongminh Date: 2007-08-22 08:50:02 +0000 (Wed, 22 Aug 2007) Log Message: ----------- Dutch translation fixes by Waddr. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-22 08:38:50 UTC (rev 4085) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-22 08:50:02 UTC (rev 4086) @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ This bot is used for checking external links found at the wiki. It checks several pages at once, with a limit set by the config variable @@ -101,7 +101,7 @@ talk_report_archive = { 'de': u'Die Webseite wurde vom Internet Archive gespeichert. Bitte verlinke gegebenenfalls eine geeignete archivierte Version: [%s]. ', 'en': u'\nThe web page has been saved by the Internet Archive. Please consider linking to an appropriate archived version: [%s]. ', - 'nl': u'\nDeze website is bewaard in het Internet Archief. Overweeg te linken naar een gearchiveerde pagina: [%s]. ', + 'nl': u'\nDeze website is bewaard in het Internet Archive. Overweeg te linken naar een gearchiveerde pagina: [%s]. ', 'no': u'\nDenne nettsiden er lagra i Internet Archive. Vurder om lenka kan endres til å peke til en av de arkiverte versjonene: [%s]. ', 'pt': u'Esta página web foi gravada na Internet Archive. Por favor considere o link para a versão arquivada: [%s]. ', }

1 0

[Pywikipedia-l] SVN: [4085] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 22 Aug '07

22 Aug '07

Revision: 4085 Author: cosoleto Date: 2007-08-22 08:38:50 +0000 (Wed, 22 Aug 2007) Log Message: ----------- code cleanup Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-08-21 23:52:43 UTC (rev 4084) +++ trunk/pywikipedia/copyright.py 2007-08-22 08:38:50 UTC (rev 4085) @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- """ -This robot checks copyright text in Google and Yahoo. +This robot checks copyright text in Google, Yahoo and Live Search. Google search requires to install the pyGoogle module from http://pygoogle.sf.net and get a Google API license key from @@ -11,6 +11,9 @@ Yahoo! search requires pYsearch module from http://pysearch.sourceforge.net and a Yahoo AppID from http://developer.yahoo.com. +Windows Live Search requires to install the SOAPpy module from +http://pywebsvcs.sf.net and get an AppID from http://search.msn.com/developer. + You can run the bot with the following commandline parameters: -g - Use Google search engine @@ -182,7 +185,7 @@ sections_to_skip = { 'en':['References', 'Further reading', 'Citations', 'External links'], - 'it':['Bibliografia', 'Riferimenti bibliografici', "Collegamenti esterni"], + 'it':['Bibliografia', 'Riferimenti bibliografici', 'Collegamenti esterni', 'Pubblicazioni principali'], } def skip_section(text): @@ -222,23 +225,27 @@ def load_pages(force_update = False): for page, path in exclusion_file_list(): try: - length = 0 - length = os.path.getsize(path) - file_age = time.time() - os.path.getmtime(path) - if file_age > 24 * 60 * 60: - print 'Updating page [[' + page.title() + ']] to exclude new URLs...' - length = 0 + if not os.path.exists(path): + print 'Creating file \'%s\' ([[%s]])' % (path, page.title()) + force_update = True + else: + file_age = time.time() - os.path.getmtime(path) + if file_age > 24 * 60 * 60: + print 'Updating file \'%s\' ([[%s]])' % (path, page.title()) + force_update = True except OSError: - pass + raise - if length == 0 or force_update: + if force_update: try: data = page.get() f = codecs.open(path, 'w', 'utf-8') f.write(data) f.close() - except wikipedia.IsRedirectPage: - data = page.get(get_redirect=True) + except KeyboardInterrupt: + raise + except wikipedia.IsRedirectPage, arg: + data = wikipedia.Page(page.site(), arg).get() except: print 'Getting page failed' return @@ -666,13 +673,13 @@ for arg in wikipedia.handleArgs(): #if arg.startswith('-repeat'): # repeat = True - if arg.startswith('-y'): + if arg == '-y': config.copyright_yahoo = True - elif arg.startswith('-g'): + elif arg == '-g': config.copyright_google = True - elif arg.startswith('-ny'): + elif arg == '-ny': config.copyright_yahoo = False - elif arg.startswith('-ng'): + elif arg == '-ng': config.copyright_google = False elif arg.startswith('-output'): if len(arg) >= 8:

1 0

[Pywikipedia-l] Get lost
by Andre Engels 22 Aug '07

22 Aug '07

WHY DON'T YOU JUST ALL GET LOST?????????????????????????????? -- Andre Engels, andreengels(a)gmail.com ICQ: 6260644 -- Skype: a_engels

2 1

[Pywikipedia-l] SVN: [4083] trunk/pywikipedia/rcsort.py
by a_engels＠svn.wikimedia.org 21 Aug '07

21 Aug '07

Revision: 4083 Author: a_engels Date: 2007-08-21 23:44:54 +0000 (Tue, 21 Aug 2007) Log Message: ----------- update - nl: now uses localized names for special pages Modified Paths: -------------- trunk/pywikipedia/rcsort.py Modified: trunk/pywikipedia/rcsort.py =================================================================== --- trunk/pywikipedia/rcsort.py 2007-08-21 22:27:47 UTC (rev 4082) +++ trunk/pywikipedia/rcsort.py 2007-08-21 23:44:54 UTC (rev 4083) @@ -1,11 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/python # -*- coding: utf-8 -*- # A tool to see the recentchanges ordered by user instead of by date. This # is meant to be run as a CGI script. # Currently only works on Dutch Wikipedia, I do intend to make it more generally # usable. # Permission has been asked to run this on the toolserver. -__version__ = '$Id$' import cgi import cgitb @@ -26,9 +25,8 @@ import wikipedia print "-->" mysite = wikipedia.getSite() -special = mysite.family.special_namespace(mysite.lang) -post = 'title=%s:Recentchanges' % special +post = 'title=Speciaal:RecenteWijzigingen' for element in form: post += '&%s=%s'%(element,form[element].value) if not 'limit' in form: @@ -38,7 +36,7 @@ text = text.split('\n') rcoptions = False lines = [] -Ruser = re.compile('title=\"%s\:Contributions\/([^\"]*)\"' % special) +Ruser = re.compile('title=\"Speciaal\:Bijdragen\/([^\"]*)\"') Rnumber = re.compile('tabindex=\"(\d*)\"') count = 0 for line in text: @@ -50,9 +48,8 @@ user = None count += 1 lines.append((user,count,line)) - print elif line.find('rcoptions') > -1: - print line.replace("/w/index.php?title=%s:Recentchanges&" % special,"rcsort.py?") + print line.replace("/w/index.php?title=Speciaal:RecenteWijzigingen&","rcsort.py?") rcoptions = True lines.sort() last = 0 @@ -63,10 +60,10 @@ if line[0] == None: print "<h2>Gebruiker onbekend</h2>" else: - wikipedia.output(u"<h2>%s</h2>"%line[0], toStdout=True) + wikipedia.output(u"<h2>%s</h2>"%line[0],showcgi=True) print "<ul>" last = line[0] - wikipedia.output(line[2].replace('href="/w','href="http://nl.wikipedia.org/w'), toStdout = True) + wikipedia.output(line[2].replace('href="/w','href="http://nl.wikipedia.org/w'), showcgi = True) print print "</ul>"

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

pywikibot