Pywikipedia-svn April 2009

pywikipedia-svn@lists.wikimedia.org

8 participants
162 discussions

SVN: [6650] branches/rewrite/pywikibot
by russblau＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6650 Author: russblau Date: 2009-04-21 18:37:01 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Merge family files from trunk Modified Paths: -------------- branches/rewrite/pywikibot/families/i18n_family.py branches/rewrite/pywikibot/families/wikiquote_family.py branches/rewrite/pywikibot/families/wikiversity_family.py branches/rewrite/pywikibot/family.py Property Changed: ---------------- branches/rewrite/pywikibot/families/ branches/rewrite/pywikibot/family.py Property changes on: branches/rewrite/pywikibot/families ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/pywikipedia/families:5896-6197,6212-6560 + /trunk/pywikipedia/families:5896-6197,6212-6649 Modified: branches/rewrite/pywikibot/families/i18n_family.py =================================================================== --- branches/rewrite/pywikibot/families/i18n_family.py 2009-04-21 14:10:11 UTC (rev 6649) +++ branches/rewrite/pywikibot/families/i18n_family.py 2009-04-21 18:37:01 UTC (rev 6650) @@ -4,7 +4,7 @@ from pywikibot import family -# The Wikimedia i18n family (should be called Betawiki, but already exists) +# The Wikimedia i18n family class Family(family.Family): Modified: branches/rewrite/pywikibot/families/wikiquote_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikiquote_family.py 2009-04-21 14:10:11 UTC (rev 6649) +++ branches/rewrite/pywikibot/families/wikiquote_family.py 2009-04-21 18:37:01 UTC (rev 6650) @@ -52,17 +52,17 @@ # languages are put in the order given there. All other languages are put # after those, in code-alphabetical order. - alphabetic = ['af','ar','roa-rup','om','bg','be','bn','bs', - 'ca','chr','co','cs','cy','da','de','als','et', - 'el','en','es','eo','eu','fa','fr','fy','ga','gv', - 'gd','gl','ko','hi','hr','io','id','ia','is','it', - 'he','jv','ka','csb','ks','sw','la','lt','hu', - 'mk','mg','ml','mi','mr','zh-cfr','mn','nah','na', - 'nl','ja','no','nb','oc','nds','pl','pt','ro','ru', - 'sa','st','sq','si','simple','sk','sl','sr','su', - 'fi','sv','ta','tt','th','ur','vi', - 'tpi','tr','uk','vo','yi','yo','za','zh','zh-cn', - 'zh-tw'] + alphabetic = ['af','am','ang','ar','roa-rup','ast','az','bn', + 'zh-min-nan','bg','be','bs','br','ca','chr','co','cs','cy', + 'da','de','als','et','el','en','es','eo','eu','fa','fr', + 'fy','ga','gv','gu','gd','gl','ko','hy','hi','hr','io', + 'id','ia','is','it','he','jv','kn','ka','ks','csb','kk', + 'ky','sw','ku','la','lb','lt','li','hu','mk','mg','ml', + 'mi','mr','zh-cfr','mn','nah','na','nl','ja','no','nb', + 'nn','oc','om','nds','uz','pl','pt','ro','ru','sa','st', + 'sq','si','simple','sk','sl','sr','su','fi','sv','ta','tt', + 'te','th','ur','vi','tpi','tr','uk','vo','yi','yo','wo', + 'za','zh','zh-cn','zh-tw'] self.interwiki_putfirst = { 'en': alphabetic, Modified: branches/rewrite/pywikibot/families/wikiversity_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikiversity_family.py 2009-04-21 14:10:11 UTC (rev 6649) +++ branches/rewrite/pywikibot/families/wikiversity_family.py 2009-04-21 18:37:01 UTC (rev 6650) @@ -12,7 +12,7 @@ self.name = 'wikiversity' self.languages_by_size = [ - 'en', 'fr', 'es', 'de', 'it', 'el', 'cs', 'ja', 'pt', + 'en', 'fi', 'fr', 'es', 'de', 'it', 'el', 'cs', 'ja', 'pt' ] self.langs = { Modified: branches/rewrite/pywikibot/family.py =================================================================== --- branches/rewrite/pywikibot/family.py 2009-04-21 14:10:11 UTC (rev 6649) +++ branches/rewrite/pywikibot/family.py 2009-04-21 18:37:01 UTC (rev 6650) @@ -722,7 +722,7 @@ 'nn': [u'omdiriger'], 'oc': [u'redireccion'], 'pdc': [u'weiterleitung'], - 'pl': [u'redirect', u'tam', u'przekieruj'], + 'pl': [u'redirect', u'patrz', u'tam', u'przekieruj'], 'ro': [u'redirecteaza'], 'ru': [u'redirect', u'перенаправление', u'перенапр'], 'sa': [u'#पुनर्निदेशन'], Property changes on: branches/rewrite/pywikibot/family.py ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/pywikipedia/family.py:6057-6560 + /trunk/pywikipedia/family.py:6057-6649

1 0

SVN: [6649] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6649 Author: cosoleto Date: 2009-04-21 14:10:11 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Fix regression introduced from r6600 when the -text command line parameter is used. Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2009-04-21 14:01:36 UTC (rev 6648) +++ trunk/pywikipedia/copyright.py 2009-04-21 14:10:11 UTC (rev 6649) @@ -1191,7 +1191,7 @@ wikipedia.output(__doc__, 'utf-8') if text: - output = query(lines = text.splitlines()) + output = SearchEngine().query(lines = text.splitlines()) if output: wikipedia.output(output)

1 0

SVN: [6648] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6648 Author: cosoleto Date: 2009-04-21 14:01:36 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Removed old 'cleanwikicode' function, use 'remove_wikicode' instead. Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2009-04-21 11:01:08 UTC (rev 6647) +++ trunk/pywikipedia/copyright.py 2009-04-21 14:01:36 UTC (rev 6648) @@ -477,9 +477,6 @@ reWikipediaC = re.compile('(' + '|'.join(wikipedia_names.values()) + ')', re.I) reSectionNamesC = re.compile('(' + '|'.join(editsection_names.values()) + ')') -def cleanwikicode(text): - remove_wikicode(text) - def remove_wikicode(text, re_dotall = False, remove_quote = exclude_quote, debug = False): if not text: return ""

1 0

SVN: [6647] trunk/pywikipedia/copyright.py
by cosoleto＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6647 Author: cosoleto Date: 2009-04-21 11:01:08 +0000 (Tue, 21 Apr 2009) Log Message: ----------- A not complete prettification try, instead of rewrite from scratch. Minor functional changes. Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2009-04-21 06:57:10 UTC (rev 6646) +++ trunk/pywikipedia/copyright.py 2009-04-21 11:01:08 UTC (rev 6647) @@ -250,8 +250,6 @@ 'zh':[u'參考文獻',u'参考文献',u'參考資料',u'参考资料', u'資料來源',u'资料来源',u'參見',u'参见',u'參閱',u'参阅'], } -num_google_queries = 0 ; num_yahoo_queries = 0 ; num_msn_queries = 0 - if enable_color: warn_color = '\03{%s}' % warn_color error_color = '\03{%s}' % error_color @@ -271,15 +269,6 @@ def error(text ,prefix = None): _output(text, prefix = prefix, color = error_color) -def print_stats(): - wikipedia.output('\n' - 'Search engine | number of queries\n' - '---------------------------------\n' - 'Google | %s\n' - 'Yahoo! | %s\n' - 'Live Search | %s\n' - % (num_google_queries, num_yahoo_queries, num_msn_queries)) - def skip_section(text): l = list() for s in sections_to_skip.values(): @@ -307,107 +296,127 @@ return text[:start.start()] return text -def exclusion_file_list(): - for i in pages_for_exclusion_database: - path = wikipedia.config.datafilepath(appdir, i[0], i[2]) - wikipedia.config.makepath(path) - p = wikipedia.Page(wikipedia.getSite(i[0]), i[1]) - yield p, path +class URLExclusion: + def __init__(self): + self.URLlist = set() + self.scan() -def load_pages(force_update = False): - for page, path in exclusion_file_list(): - try: - force_load = force_update - if not os.path.exists(path): - print 'Creating file \'%s\' (%s)' % ( - wikipedia.config.shortpath(path), page.aslink()) - force_load = True - else: - file_age = time.time() - os.path.getmtime(path) - if file_age > 24 * 60 * 60: - print 'Updating file \'%s\' (%s)' % ( - wikipedia.config.shortpath(path), page.aslink()) - force_load = True - except OSError: - raise + def pages_list(self): + for i in pages_for_exclusion_database: + path = wikipedia.config.datafilepath(appdir, i[0], i[2]) + wikipedia.config.makepath(path) + page = wikipedia.Page(wikipedia.getSite(i[0]), i[1]) + yield page, path - if force_load: - data = None + def download(self, force_update = False): + for page, path in self.pages_list(): + download = force_update try: - data = page.get() - except KeyboardInterrupt: + if not os.path.exists(path): + print 'Creating file \'%s\' (%s)' % (wikipedia.config.shortpath(path), + page.aslink()) + download = True + else: + file_age = time.time() - os.path.getmtime(path) + if download or file_age > 24 * 60 * 60: + print 'Updating file \'%s\' (%s)' % ( + wikipedia.config.shortpath(path), page.aslink()) + download = True + except OSError: raise - except wikipedia.IsRedirectPage, arg: - data = page.getRedirectTarget().get() - except: - error('Getting page failed') - if data: - f = codecs.open(path, 'w', 'utf-8') - f.write(data) - f.close() - return + if download: + data = None + try: + data = page.get() + except KeyboardInterrupt: + raise + except wikipedia.IsRedirectPage, arg: + data = page.getRedirectTarget().get() + except: + error('Getting page failed') -def check_list(url, clist, verbose = False): - for entry in clist: - if entry: - if url.find(entry) != -1: - if verbose > 1: - warn('URL Excluded: %s\nReason: %s' % (url, entry)) - elif verbose: - warn('URL Excluded: %s' % url) - return True + if data: + f = codecs.open(path, 'w', 'utf-8') + f.write(data) + f.close() + + def update(self): + self.download(force_update = True) + self.scan() -def exclusion_list(): - prelist = [] - result_list = [] - load_pages() + def check(self, url, verbose = False): + for entry in self.URLlist: + if url.find(entry) != -1: + if verbose > 1: + warn('URL Excluded: %s\nReason: %s' % (url, entry)) + elif verbose: + warn('URL Excluded: %s' % url) + return True + return False - for page, path in exclusion_file_list(): - if 'exclusion_list.txt' in path: - result_list += re.sub("</?pre>","", - read_file(path, - cut_comment=True, - cut_newlines=True) - ).splitlines() - else: - data = read_file(path) - # wikipedia:en:Wikipedia:Mirrors and forks - prelist += re.findall("(?i)url\s*=\s*<nowiki>(?:http://)?(.*)</nowiki>", data) - prelist += re.findall("(?i)\*\s*Site:\s*\[?(?:http://)?(.*)\]?", data) - # wikipedia:it:Wikipedia:Cloni - if 'it/Cloni.txt' in path: - prelist += re.findall('(?mi)^==(?!=)\s*\[?\s*(?:<nowiki>)?\s*(?:http://)?(.*?)(?:</nowiki>)?\s*\]?\s*==', data) - list1 = [] - for entry in prelist: - list1 += entry.split(", ") - list2 = [] - for entry in list1: - list2 += entry.split("and ") - for entry in list2: - # Remove unnecessary part of URL - entry = re.sub("http://", "", entry) - entry = re.sub("www\.", "", entry) - entry = re.sub("</?nowiki>", "", entry) - if entry: - if '/' in entry: - entry = entry[:entry.rfind('/')] + def scan(self): + prelist = [] ; result_list = [] + self.download() - entry = re.sub("\s.*", "", entry) + for page, path in self.pages_list(): + if 'exclusion_list.txt' in path: + result_list += re.sub("</?pre>","", + read_file(path, + cut_comment=True, + cut_newlines=True) + ).splitlines() + else: + data = read_file(path) + # wikipedia:en:Wikipedia:Mirrors and forks + prelist += re.findall("(?i)url\s*=\s*<nowiki>(?:http://)?(.*)</nowiki>", data) + prelist += re.findall("(?i)\*\s*Site:\s*\[?(?:http://)?(.*)\]?", data) + # wikipedia:it:Wikipedia:Cloni + if 'it/Cloni.txt' in path: + prelist += re.findall('(?mi)^==(?!=)\s*\[?\s*(?:<nowiki>)?\s*(?:http://)?(.*?)(?:</nowiki>)?\s*\]?\s*==', data) + list1 = [] + for entry in prelist: + list1 += entry.split(", ") + list2 = [] + for entry in list1: + list2 += entry.split("and ") + for entry in list2: + # Remove unnecessary part of URL + entry = re.sub("http://", "", entry) + entry = re.sub("www\.", "", entry) + entry = re.sub("</?nowiki>", "", entry) + if entry: + if '/' in entry: + entry = entry[:entry.rfind('/')] - if len(entry) > 4: - result_list.append(entry) + entry = re.sub("\s.*", "", entry) - result_list += read_file( - wikipedia.config.datafilepath(appdir, 'exclusion_list.txt'), - cut_comment = True, cut_newlines = True - ).splitlines() + if len(entry) > 4: + result_list.append(entry) - for i in range(len(result_list)): - result_list[i] = re.sub('\s+$', '', result_list[i]) + result_list += read_file( + wikipedia.config.datafilepath(appdir, 'exclusion_list.txt'), + cut_comment = True, cut_newlines = True + ).splitlines() - return result_list + for i in range(len(result_list)): + cleaned = re.sub('\s+$', '', result_list[i]) + if cleaned: + self.URLlist.add(cleaned) + def sanity_check(self): + print "Exclusion list sanity check..." + for entry in self.URLlist: + if (not '.' in entry and not '/' in entry) or len(entry) < 5: + print "** " + entry + + def dump(self): + f = open(wikipedia.config.datafilepath(appdir, 'exclusion_list.dump'), 'w') + f.write('\n'.join(self.URLlist)) + f.close() + print "Exclusion list dump saved." + + def read_file(filename, cut_comment = False, cut_newlines = False): text = u"" @@ -565,18 +574,6 @@ return text -def exclusion_list_sanity_check(): - print "Exclusion list sanity check..." - for entry in excl_list: - if (not '.' in entry and not '/' in entry) or len(entry) < 5: - print "** " + entry - -def exclusion_list_dump(): - f = open(wikipedia.config.datafilepath(appdir, 'exclusion_list.dump'), 'w') - f.write('\n'.join(excl_list)) - f.close() - print "Exclusion list dump saved." - def n_index(text, n, sep): pos = 0 while n>0: @@ -603,73 +600,270 @@ break return l -def query(lines = [], max_query_len = 1300, wikicode = True): - # Google max_query_len = 1480? - # - '-Wikipedia ""' = 1467 +class SearchEngine: - # Google limit queries to 32 words. + num_google_queries = num_yahoo_queries = num_msn_queries = 0 - output = u"" - n_query = 0 - previous_group_url = 'none' + def __init__(self): + self.URLexcl = URLExclusion() - for line in lines: - if wikicode: - line = remove_wikicode(line) - for search_words in mysplit(line, number_of_words, " "): - if len(search_words) > min_query_string_len: - if config.copyright_economize_query: - if economize_query(search_words): - warn(search_words, prefix = 'Text excluded') + def __del__(self): + self.print_stats() + + def query(self, lines = [], max_query_len = 1300, wikicode = True): + # Google max_query_len = 1480? + # - '-Wikipedia ""' = 1467 + + # Google limit queries to 32 words. + + n_query = 0 + output = unicode() + previous_group_url = 'null' + + for line in lines: + if wikicode: + line = remove_wikicode(line) + for search_words in mysplit(line, number_of_words, " "): + if len(search_words) > min_query_string_len: + if config.copyright_economize_query: + if economize_query(search_words): + warn(search_words, prefix = 'Text excluded') + consecutive = False + continue + n_query += 1 + #wikipedia.output(search_words) + if config.copyright_max_query_for_page and n_query > config.copyright_max_query_for_page: + warn(u"Max query limit for page reached") + return output + if config.copyright_skip_query > n_query: + continue + if len(search_words) > max_query_len: + search_words = search_words[:max_query_len] consecutive = False - continue - n_query += 1 - #wikipedia.output(search_words) - if config.copyright_max_query_for_page and n_query > config.copyright_max_query_for_page: - warn(u"Max query limit for page reached") - return output - if config.copyright_skip_query > n_query: - continue - if len(search_words) > max_query_len: - search_words = search_words[:max_query_len] - consecutive = False - if " " in search_words: - search_words = search_words[:search_words.rindex(" ")] + if " " in search_words: + search_words = search_words[:search_words.rindex(" ")] - results = get_results(search_words) + results = self.get_results(search_words) - group_url = '' ; cmp_group_url = '' + group_url = '' ; cmp_group_url = '' - for url, engine, comment in results: - if comment: - group_url += '\n*%s - %s (%s)' % (engine, url, "; ".join(comment)) - else: - group_url += '\n*%s - %s' % (engine, url) - cmp_group_url += '\n*%s - %s' % (engine, url) - if results: - group_url_list = group_url.splitlines() - cmp_group_url_list = cmp_group_url.splitlines() - group_url_list.sort() - cmp_group_url_list.sort() - group_url = '\n'.join(group_url_list) - cmp_group_url = '\n'.join(cmp_group_url_list) - if previous_group_url == cmp_group_url: - if consecutive: - output += ' ' + search_words + for url, engine, comment in results: + if comment: + group_url += '\n*%s - %s (%s)' % (engine, url, "; ".join(comment)) else: - output += '\n**' + search_words + group_url += '\n*%s - %s' % (engine, url) + cmp_group_url += '\n*%s - %s' % (engine, url) + if results: + group_url_list = group_url.splitlines() + cmp_group_url_list = cmp_group_url.splitlines() + group_url_list.sort() + cmp_group_url_list.sort() + group_url = '\n'.join(group_url_list) + cmp_group_url = '\n'.join(cmp_group_url_list) + if previous_group_url == cmp_group_url: + if consecutive: + output += ' ' + search_words + else: + output += '\n**' + search_words + else: + output += group_url + '\n**' + search_words + + previous_group_url = cmp_group_url + consecutive = True else: - output += group_url + '\n**' + search_words + consecutive = False + else: + consecutive = False - previous_group_url = cmp_group_url - consecutive = True + return output + + def add_in_urllist(self, url, add_item, engine, cache_url = None): + if (engine == 'google' and config.copyright_check_in_source_google) or \ + (engine == 'yahoo' and config.copyright_check_in_source_yahoo) or \ + (engine == 'msn' and config.copyright_check_in_source_msn): + check_in_source = True + else: + check_in_source = False + + if check_in_source or config.copyright_show_date or config.copyright_show_length: + s = None + cache = False + + # list to store date, length, cache URL + comment = list() + + try: + s = WebPage(add_item, self.URLexcl) + except URL_exclusion: + pass + except NoWebPage: + cache = True + + if s: + # Before of add url in result list, perform the check in source + if check_in_source: + if s.check_in_source(): + return + + if config.copyright_show_date: + date = s.lastmodified() + if date: + if date[:3] != time.localtime()[:3]: + comment.append("%s/%s/%s" % (date[2], date[1], date[0])) + + unit = 'bytes' + + if config.copyright_show_length: + length = s.length() + if length > 1024: + # convert in kilobyte + length /= 1024 + unit = 'KB' + if length > 1024: + # convert in megabyte + length /= 1024 + unit = 'MB' + if length > 0: + comment.append("%d %s" % (length, unit)) + + if cache: + if cache_url: + if engine == 'google': + comment.append('[http://www.google.com/search?sourceid=navclient&q=cache:%s Google cache]' % urllib.quote(short_url(add_item))) + elif engine == 'yahoo': + #cache = False + #comment.append('[%s Yahoo cache]' % re.sub('&appid=[^&]*','', urllib2.unquote(cache_url))) + comment.append("''Yahoo cache''") + elif engine == 'msn': + comment.append('[%s Live cache]' % re.sub('&lang=[^&]*','', cache_url)) else: - consecutive = False - else: - consecutive = False + comment.append('[http://web.archive.org/*/%s archive.org]' % short_url(add_item)) - return output + for i in range(len(url)): + if add_item in url[i]: + if engine not in url[i][1]: + if url[i][2]: + comment = url[i][2] + url[i] = (add_item, url[i][1] + ', ' + engine, comment) + return + url.append((add_item, engine, comment)) + return + def soap(self, engine, query, url, numresults = 10): + print " %s query..." % engine.capitalize() + search_request_retry = config.copyright_connection_tries + query_success = False + + while search_request_retry: + try: + if engine == 'google': + import google + google.LICENSE_KEY = config.google_key + data = google.doGoogleSearch('%s "%s"' % (no_result_with_those_words, query)) + for entry in data.results: + self.add_in_urllist(url, entry.URL, 'google', entry.cachedSize) + + self.num_google_queries += 1 + + elif engine == 'yahoo': + import yahoo.search.web + data = yahoo.search.web.WebSearch(config.yahoo_appid, query='"%s" %s' % ( + query.encode('utf_8'), + no_result_with_those_words + ), results = numresults) + for entry in data.parse_results(): + cacheurl = None + if entry.Cache: + cacheurl = entry.Cache.Url + self.add_in_urllist(url, entry.Url, 'yahoo', cacheurl) + + self.num_yahoo_queries += 1 + + elif engine == 'msn': + #max_query_len = 150? + from SOAPpy import WSDL + + try: + server = WSDL.Proxy('http://soap.search.msn.com/webservices.asmx?wsdl') + except Exception, err: + error("Live Search Error: %s" % err) + raise + + params = {'AppID': config.msn_appid, 'Query': '%s "%s"' % (no_result_with_those_words, query), + 'CultureInfo': region_code, 'SafeSearch': 'Off', 'Requests': { + 'SourceRequest':{'Source': 'Web', 'Offset': 0, 'Count': 10, 'ResultFields': 'All',}}} + + results = '' + + server_results = server.Search(Request = params) + if server_results.Responses[0].Results: + results = server_results.Responses[0].Results[0] + if results: + # list or instance? + if type(results) == type([]): + for entry in results: + cacheurl = None + if hasattr(entry, 'CacheUrl'): + cacheurl = entry.CacheUrl + self.add_in_urllist(url, entry.Url, 'msn', cacheurl) + else: + cacheurl = None + if hasattr(results, 'CacheUrl'): + cacheurl = results.CacheUrl + self.add_in_urllist(url, results.Url, 'msn', cacheurl) + + self.num_msn_queries += 1 + + search_request_retry = 0 + query_success = True + except KeyboardInterrupt: + raise + except Exception, err: + # Something is going wrong... + if 'Daily limit' in str(err) or 'Insufficient quota for key' in str(err): + exceeded_in_queries('google') + elif 'limit exceeded' in str(err): + exceeded_in_queries('yahoo') + elif 'Invalid value for AppID in request' in str(err): + exceeded_in_queries('msn') + else: + raise + error(err, "Got an error") + + if search_request_retry: + search_request_retry -= 1 + + if not query_success: + error('No response for: %s' % query, "Error (%s)" % engine) + + def get_results(self, query, numresults = 10): + result_list = list() + query = re.sub("[()\"<>]", "", query) + # wikipedia.output(query) + if config.copyright_google: + self.soap('google', query, result_list) + if config.copyright_yahoo: + self.soap('yahoo', query, result_list, numresults = numresults) + if config.copyright_msn: + self.soap('msn', query, result_list) + + offset = 0 + for i in range(len(result_list)): + if self.URLexcl.check(result_list[i + offset][0], verbose = True): + result_list.pop(i + offset) + offset += -1 + return result_list + + def print_stats(self): + wikipedia.output('\n' + 'Search engine | number of queries\n' + '---------------------------------\n' + 'Google | %s\n' + 'Yahoo! | %s\n' + 'Live Search | %s\n' + % (self.num_google_queries, self.num_yahoo_queries, + self.num_msn_queries)) + source_seen = set() positive_source_seen = set() @@ -683,12 +877,13 @@ """ """ - def __init__(self, url): + def __init__(self, url, URLExcl): """ """ global source_seen + self.URLexcludedlist = URLExcl.URLlist - if url in source_seen or check_list(url, excl_list): + if url in source_seen or URLExcl.check(url): raise URL_exclusion self._url = url @@ -746,8 +941,8 @@ def check_regexp(self, reC, text, filename = None): m = reC.search(text) if m: - global excl_list, positive_source_seen - excl_list += [self._url] + global positive_source_seen + self.URLexcludedlist.add(self._url) positive_source_seen.add(self._url) if filename: write_log("%s (%s)\n" % (self._url, m.group()), filename) @@ -796,79 +991,7 @@ source_seen.add(self._url) return False -def add_in_urllist(url, add_item, engine, cache_url = None): - if (engine == 'google' and config.copyright_check_in_source_google) or \ - (engine == 'yahoo' and config.copyright_check_in_source_yahoo) or \ - (engine == 'msn' and config.copyright_check_in_source_msn): - check_in_source = True - else: - check_in_source = False - - if check_in_source or config.copyright_show_date or config.copyright_show_length: - s = None - cache = False - - # list to store date, length, cache URL - comment = list() - - try: - s = WebPage(add_item) - except URL_exclusion: - pass - except NoWebPage: - cache = True - - if s: - # Before of add url in result list, perform the check in source - if check_in_source: - if s.check_in_source(): - return - - if config.copyright_show_date: - date = s.lastmodified() - if date: - if date[:3] != time.localtime()[:3]: - comment.append("%s/%s/%s" % (date[2], date[1], date[0])) - - unit = 'bytes' - - if config.copyright_show_length: - length = s.length() - if length > 1024: - # convert in kilobyte - length /= 1024 - unit = 'KB' - if length > 1024: - # convert in megabyte - length /= 1024 - unit = 'MB' - if length > 0: - comment.append("%d %s" % (length, unit)) - - if cache: - if cache_url: - if engine == 'google': - comment.append('[http://www.google.com/search?sourceid=navclient&q=cache:%s Google cache]' % urllib.quote(short_url(add_item))) - elif engine == 'yahoo': - #cache = False - #comment.append('[%s Yahoo cache]' % re.sub('&appid=[^&]*','', urllib2.unquote(cache_url))) - comment.append("''Yahoo cache''") - elif engine == 'msn': - comment.append('[%s Live cache]' % re.sub('&lang=[^&]*','', cache_url)) - else: - comment.append('[http://web.archive.org/*/%s archive.org]' % short_url(add_item)) - - for i in range(len(url)): - if add_item in url[i]: - if engine not in url[i][1]: - if url[i][2]: - comment = url[i][2] - url[i] = (add_item, url[i][1] + ', ' + engine, comment) - return - url.append((add_item, engine, comment)) - return - def exceeded_in_queries(engine): """Behavior if an exceeded error occur.""" @@ -883,112 +1006,6 @@ if config.copyright_exceeded_in_queries == 3: raise 'Got a queries exceeded error.' -def soap(engine, query, url, numresults = 10): - global num_google_queries, num_yahoo_queries, num_msn_queries - - print " %s query..." % engine.capitalize() - search_request_retry = config.copyright_connection_tries - query_success = False - - while search_request_retry: - try: - if engine == 'google': - import google - google.LICENSE_KEY = config.google_key - data = google.doGoogleSearch('%s "%s"' % (no_result_with_those_words, query)) - for entry in data.results: - add_in_urllist(url, entry.URL, 'google', entry.cachedSize) - - num_google_queries += 1 - - elif engine == 'yahoo': - import yahoo.search.web - data = yahoo.search.web.WebSearch(config.yahoo_appid, query='"%s" %s' % ( - query.encode('utf_8'), - no_result_with_those_words - ), results = numresults) - for entry in data.parse_results(): - cacheurl = None - if entry.Cache: - cacheurl = entry.Cache.Url - add_in_urllist(url, entry.Url, 'yahoo', cacheurl) - - num_yahoo_queries += 1 - - elif engine == 'msn': - #max_query_len = 150? - from SOAPpy import WSDL - - try: - server = WSDL.Proxy('http://soap.search.msn.com/webservices.asmx?wsdl') - except Exception, err: - error("Live Search Error: %s" % err) - raise - - params = {'AppID': config.msn_appid, 'Query': '%s "%s"' % (no_result_with_those_words, query), - 'CultureInfo': region_code, 'SafeSearch': 'Off', 'Requests': { - 'SourceRequest':{'Source': 'Web', 'Offset': 0, 'Count': 10, 'ResultFields': 'All',}}} - - results = '' - - server_results = server.Search(Request = params) - if server_results.Responses[0].Results: - results = server_results.Responses[0].Results[0] - if results: - # list or instance? - if type(results) == type([]): - for entry in results: - cacheurl = None - if hasattr(entry, 'CacheUrl'): - cacheurl = entry.CacheUrl - add_in_urllist(url, entry.Url, 'msn', cacheurl) - else: - cacheurl = None - if hasattr(results, 'CacheUrl'): - cacheurl = results.CacheUrl - add_in_urllist(url, results.Url, 'msn', cacheurl) - - num_msn_queries += 1 - - search_request_retry = 0 - query_success = True - except KeyboardInterrupt: - raise - except Exception, err: - # Something is going wrong... - if 'Daily limit' in str(err) or 'Insufficient quota for key' in str(err): - exceeded_in_queries('google') - elif 'limit exceeded' in str(err): - exceeded_in_queries('yahoo') - elif 'Invalid value for AppID in request' in str(err): - exceeded_in_queries('msn') - else: - error(err, "Got an error") - - if search_request_retry: - search_request_retry -= 1 - - if not query_success: - error('No response for: %s' % query, "Error (%s)" % engine) - -def get_results(query, numresults = 10): - result_list = list() - query = re.sub("[()\"<>]", "", query) - # wikipedia.output(query) - if config.copyright_google: - soap('google', query, result_list) - if config.copyright_yahoo: - soap('yahoo', query, result_list, numresults = numresults) - if config.copyright_msn: - soap('msn', query, result_list) - - offset = 0 - for i in range(len(result_list)): - if check_list(result_list[i + offset][0], excl_list, verbose = True): - result_list.pop(i + offset) - offset += -1 - return result_list - def get_by_id(title, id): return wikipedia.getSite().getUrl("/w/index.php?title=%s&oldid=%s&action=raw" % (title, id)) @@ -1011,6 +1028,7 @@ """ """ self.generator = generator + self.SearchEngine = SearchEngine() def run(self): """ @@ -1048,7 +1066,7 @@ if remove_wikicode_dotall: text = remove_wikicode(text, re_dotall = True) - output = query(lines = text.splitlines(), wikicode = not remove_wikicode_dotall) + output = self.SearchEngine.query(lines = text.splitlines(), wikicode = not remove_wikicode_dotall) if output: write_log('=== [[' + page.title() + ']] ===' + output + '\n', filename = output_file) @@ -1144,7 +1162,7 @@ except ValueError: namespaces.append(arg[11:]) elif arg.startswith('-forceupdate'): - load_pages(force_update = True) + URLExclusion().update() elif arg == '-repeat': repeat = True elif arg.startswith('-new'): @@ -1187,10 +1205,7 @@ preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber = pageNumber) bot = CheckRobot(preloadingGen) bot.run() - print_stats() -excl_list = exclusion_list() - if number_of_words > 22 and config.copyright_msn: warn("Live Search requires a lower value for 'number_of_words' variable " "(current value is %d, a good value may be 22)." % (number_of_words), prefix = 'Warning')

1 0

SVN: [6646] branches/rewrite/pywikibot/bot.py
by nicdumz＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6646 Author: nicdumz Date: 2009-04-21 06:57:10 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Fixing previous commit. **kargs and not kargs adding \n when using the logger (switching from a DCVS to CVS makes you look stupid ;) ) Modified Paths: -------------- branches/rewrite/pywikibot/bot.py Modified: branches/rewrite/pywikibot/bot.py =================================================================== --- branches/rewrite/pywikibot/bot.py 2009-04-21 06:39:18 UTC (rev 6645) +++ branches/rewrite/pywikibot/bot.py 2009-04-21 06:57:10 UTC (rev 6646) @@ -469,7 +469,7 @@ """ Only accepts options defined in availableOptions """ - self.setOptions(kwargs) + self.setOptions(**kwargs) def setOptions(self, **kwargs): """ @@ -478,14 +478,14 @@ # contains the options overriden from defaults self.options = {} - validOptions = set(availableOptions.iterkeys()) + validOptions = set(Bot.availableOptions.iterkeys()) receivedOptions = set(kwargs.iterkeys()) for opt in receivedOptions & validOptions: self.options[opt] = kwargs[opt] for opt in receivedOptions - validOptions: - logging.warning(u'%s is not a valid option. It was ignored' % opt) + logging.warning(u'%s is not a valid option. It was ignored\n' % opt) def getOption(self, option): """ @@ -506,7 +506,7 @@ * 'always' """ if oldtext == newtext: - logging.info(u'No changes were needed on %s' \ + pywikibot.output(u'No changes were needed on %s' \ % page.title(asLink=True)) return @@ -524,6 +524,6 @@ # Remember the choice self.options['always'] = True - if choice != 'N': + if choice != 'n': page.put(newtext, async=(choice=='a'))

1 0

SVN: [6645] branches/rewrite/pywikibot/bot.py
by nicdumz＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6645 Author: nicdumz Date: 2009-04-21 06:39:18 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Starting the Bot class: I just needed the kind of "userPut" I just implemented, instead of writing yet another version of that function. There is definitely room for improvement, please go ahead. Modified Paths: -------------- branches/rewrite/pywikibot/bot.py Modified: branches/rewrite/pywikibot/bot.py =================================================================== --- branches/rewrite/pywikibot/bot.py 2009-04-21 06:33:00 UTC (rev 6644) +++ branches/rewrite/pywikibot/bot.py 2009-04-21 06:39:18 UTC (rev 6645) @@ -451,3 +451,79 @@ level=pywikibot.STDOUT) logging.exception('showHelp:') pywikibot.output(globalHelp, level=pywikibot.STDOUT) + +class Bot(object): + """ + Generic Bot to be subclassed + """ + + # Bot configuration. + # Only the keys of the dict can be passed as init options + # The values are the default values + # Extend this in subclasses! + availableOptions = { + 'always': False, # ask for confirmation when putting a page? + } + + def __init__(self, **kwargs): + """ + Only accepts options defined in availableOptions + """ + self.setOptions(kwargs) + + def setOptions(self, **kwargs): + """ + Sets the instance options + """ + # contains the options overriden from defaults + self.options = {} + + validOptions = set(availableOptions.iterkeys()) + receivedOptions = set(kwargs.iterkeys()) + + for opt in receivedOptions & validOptions: + self.options[opt] = kwargs[opt] + + for opt in receivedOptions - validOptions: + logging.warning(u'%s is not a valid option. It was ignored' % opt) + + def getOption(self, option): + """ + Get the current value of an option. + @param option: key defined in Bot.availableOptions + """ + try: + return self.options.get(option, Bot.availableOptions[option]) + except KeyError: + raise pywikibot.Error(u'%s is not a valid bot option.' % option) + + def userPut(self, page, oldtext, newtext): + """ + Print differences, ask user for confirmation, + and puts the page if needed. + + Option used: + * 'always' + """ + if oldtext == newtext: + logging.info(u'No changes were needed on %s' \ + % page.title(asLink=True)) + return + + pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" + % page.title()) + pywikibot.showDiff(oldtext, newtext) + + choice = 'a' + if not self.getOption('always'): + choice = pywikibot.inputChoice( + u'Do you want to accept these changes?', + ['Yes', 'No', 'All'], + ['y', 'N', 'a'], 'N') + if choice == 'a': + # Remember the choice + self.options['always'] = True + + if choice != 'N': + page.put(newtext, async=(choice=='a')) +

1 0

SVN: [6644] branches/rewrite/pywikibot/bot.py
by nicdumz＠svn.wikimedia.org 21 Apr '09

21 Apr '09

Revision: 6644 Author: nicdumz Date: 2009-04-21 06:33:00 +0000 (Tue, 21 Apr 2009) Log Message: ----------- Adding an exception formatter to avoid Unicode/String mix ups in the underlying logging module. Modified Paths: -------------- branches/rewrite/pywikibot/bot.py Modified: branches/rewrite/pywikibot/bot.py =================================================================== --- branches/rewrite/pywikibot/bot.py 2009-04-20 17:37:31 UTC (rev 6643) +++ branches/rewrite/pywikibot/bot.py 2009-04-21 06:33:00 UTC (rev 6644) @@ -63,7 +63,25 @@ text = logging.handlers.RotatingFileHandler.format(self, record) return text.rstrip("\r\n") +class LoggingFormatter(logging.Formatter): + def formatException(self, ei): + """ + Make sure that the exception trace is converted to unicode: + * our pywikibot.Error traces are encoded in our + console encoding, which is needed for plainly printing them. + * but when it comes to logging using logging.exception, + the Python logging module will try to use these traces, + and it will fail if they are console encoded strings. + Formatter.formatException also strips the trailing \n, which we need. + """ + strExc = logging.Formatter.formatException(self, ei) + + if isinstance(strExc, str): + return strExc.decode(config.console_encoding) + '\n' + else: + return strExc + '\n' + def output(text, decoder=None, newline=True, toStdout=False, level=INFO): """Output a message to the user via the userinterface. @@ -207,7 +225,7 @@ else: default_handler.setLevel(INFO) default_handler.addFilter(MaxLevelFilter(INPUT)) - default_handler.setFormatter(logging.Formatter(fmt="%(message)s")) + default_handler.setFormatter(LoggingFormatter(fmt="%(message)s")) root_logger.addHandler(default_handler) # if user has enabled file logging, configure file handler @@ -220,7 +238,7 @@ filename=logfile, maxBytes=2 << 20, backupCount=5) file_handler.setLevel(DEBUG) - form = logging.Formatter( + form = LoggingFormatter( fmt="%(asctime)s %(filename)18s, %(lineno)d: " "%(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S" @@ -236,14 +254,14 @@ output_handler = TerminalHandler(strm=sys.stdout) output_handler.setLevel(STDOUT) output_handler.addFilter(MaxLevelFilter(STDOUT)) - output_handler.setFormatter(logging.Formatter(fmt="%(message)s")) + output_handler.setFormatter(LoggingFormatter(fmt="%(message)s")) root_logger.addHandler(output_handler) # handler for levels WARNING and higher warning_handler = TerminalHandler(strm=sys.stderr) warning_handler.setLevel(logging.WARNING) warning_handler.setFormatter( - logging.Formatter(fmt="%(levelname)s: %(message)s")) + LoggingFormatter(fmt="%(levelname)s: %(message)s")) root_logger.addHandler(warning_handler)

1 0

SVN: [6643] branches/rewrite/pywikibot
by russblau＠svn.wikimedia.org 20 Apr '09

20 Apr '09

Revision: 6643 Author: russblau Date: 2009-04-20 17:37:31 +0000 (Mon, 20 Apr 2009) Log Message: ----------- Fix bug in previous commit to fix Bug #2716315 (undetected session termination); if session times out while bot is operating, it will ask the user to log in again (not ideal for automated bots, but apparently unavoidable), unless you have a password file on your local machine. Modified Paths: -------------- branches/rewrite/pywikibot/data/api.py branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2009-04-20 15:29:45 UTC (rev 6642) +++ branches/rewrite/pywikibot/data/api.py 2009-04-20 17:37:31 UTC (rev 6643) @@ -229,9 +229,19 @@ self.site._userinfo.update(result['query']['userinfo']) else: self.site._userinfo = result['query']['userinfo'] - if self.site._userinfo['name'] != self.site.user(): + status = self.site._loginstatus # save previous login status + if ( ("error" in result + and result["error"]["code"].endswith("limit")) + or (status != -1 + and self.site._userinfo['name'] + != self.site._username[status])): # user is no longer logged in (session expired?) - self.site.login(self.site._username.index(self.site.user())) + # reset userinfo, then make user log in again + del self.site._userinfo + self.site._loginstatus = -1 + if status == -1: + status = 0 # default to non-sysop login + self.site.login(status) # retry the previous query continue if "warnings" in result: Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2009-04-20 15:29:45 UTC (rev 6642) +++ branches/rewrite/pywikibot/site.py 2009-04-20 17:37:31 UTC (rev 6643) @@ -615,6 +615,9 @@ self.sitelock = threading.Lock() self._msgcache = {} self.nocapitalize = self.code in self.family.nocapitalize + # _loginstatus: -1 means not logged in, 0 means logged in as user, + # 1 means logged in as sysop + self._loginstatus = -1 return # ANYTHING BELOW THIS POINT IS NOT YET IMPLEMENTED IN __init__() @@ -646,20 +649,25 @@ def login(self, sysop=False): """Log the user in if not already logged in.""" - if not hasattr(self, "_siteinfo"): - self._getsiteinfo() # check whether a login cookie already exists for this user - if hasattr(self, "_userinfo"): - if self.userinfo['name'] == self._username[sysop]: - return + if not hasattr(self, "_userinfo"): + self.getuserinfo() + if self._userinfo['name'] == self._username[sysop]: + self._loginstatus = sysop + return if not self.logged_in(sysop): loginMan = api.LoginManager(site=self, sysop=sysop, user=self._username[sysop]) if loginMan.login(retry = True): self._username[sysop] = loginMan.username + self._loginstatus = sysop if hasattr(self, "_userinfo"): del self._userinfo self.getuserinfo() + else: + self._loginstatus = -1 # failure + if not hasattr(self, "_siteinfo"): + self._getsiteinfo() forceLogin = login # alias for backward-compatibility

1 0

SVN: [6642] trunk/pywikipedia/spamremove.py
by cosoleto＠svn.wikimedia.org 20 Apr '09

20 Apr '09

Revision: 6642 Author: cosoleto Date: 2009-04-20 15:29:45 +0000 (Mon, 20 Apr 2009) Log Message: ----------- Spanish translation. Patch by KveD (#2776371). Modified Paths: -------------- trunk/pywikipedia/spamremove.py Modified: trunk/pywikipedia/spamremove.py =================================================================== --- trunk/pywikipedia/spamremove.py 2009-04-20 13:51:22 UTC (rev 6641) +++ trunk/pywikipedia/spamremove.py 2009-04-20 15:29:45 UTC (rev 6642) @@ -35,6 +35,7 @@ 'ar': u'إزالة الوصلات إلى موقع سبام %s', 'de': u'Entferne in Spam-Blacklist eingetragenen Weblink auf %s', 'en': u'Removing links to spamming site %s', + 'es': u'Removiendo enlaces a sitio publicitario %s', 'he': u'מסיר קישורים לאתר ספאם %s', 'it': u'Rimuovo link contenuto nella Spam-Blacklist %s', 'ja': u'ロボットによる: 迷惑リンク削除 %s',

1 0

SVN: [6641] trunk/pywikipedia/checkimages.py
by nicdumz＠svn.wikimedia.org 20 Apr '09

20 Apr '09

Revision: 6641 Author: nicdumz Date: 2009-04-20 13:51:22 +0000 (Mon, 20 Apr 2009) Log Message: ----------- Patch #2740955 by Terrance Rengarasu: tamil localization for checkimages.py Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-04-20 13:12:19 UTC (rev 6640) +++ trunk/pywikipedia/checkimages.py 2009-04-20 13:51:22 UTC (rev 6641) @@ -344,6 +344,7 @@ 'it':u'Progetto:Coordinamento/Immagini/Bot/WhiteTemplates', 'ja':None, 'ko': u'User:Kwjbot_IV/whitetemplates/list', + 'ta':None, 'zh':None, } @@ -353,6 +354,7 @@ 'en':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/AllowedTemplates', 'ko':u'User:Kwjbot_IV/AllowedTemplates', + 'ta':None, } # Template added when the bot finds only an hidden template and nothing else. @@ -364,7 +366,7 @@ 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Template_insufficiente|%s|__botnick__}} --~~~~", 'ja' :None, 'ko' : u"\n{{subst:User:Kwj2772/whitetemplates|%s}} --~~~~", - 'ta' : None, + 'ta' :None, 'zh' :None, } @@ -377,6 +379,7 @@ 'en' : None, 'it' : u'\n{{Progetto:Coordinamento/Immagini/Bot/Template duplicati|__images__}}', 'ja' :None, + 'ta' :None, 'zh' :None, } # Head of the message given to the author @@ -385,6 +388,7 @@ 'en' : None, 'it' : u'\n\n== File doppio ==\n', 'ja' :None, + 'ta' :None, 'zh' :None, } # Message to put in the talk @@ -393,6 +397,7 @@ 'en' : None, 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Duplicati|%s|%s|__botnick__}} --~~~~", 'ja' :None, + 'ta' :None, 'zh' :None, } # Comment used by the bot while it reports the problem in the uploader's talk @@ -401,6 +406,7 @@ 'en' : None, 'it' : u"Bot: Notifico il file doppio trovato", 'ja' :None, + 'ta' :None, 'zh' :None, } # Comment used by the bot while it reports the problem in the image @@ -409,6 +415,7 @@ 'en' : None, 'it' : u'Bot: File doppio, da cancellare', 'ja' :None, + 'ta' :None, 'zh' :None, } # Regex to detect the template put in the image's decription to find the dupe @@ -417,6 +424,7 @@ 'en' : None, 'it' : r'\{\{(?:[Tt]emplate:|)[Pp]rogetto:[Cc]oordinamento/Immagini/Bot/Template duplicati[|}]', 'ja' :None, + 'ta' :None, 'zh' :None, } # Category with the licenses and / or with subcategories with the other licenses. @@ -426,6 +434,7 @@ 'it' : 'Categoria:Template Licenze copyright', 'ja' : 'Category:画像の著作権表示テンプレート', 'ko' : '분류:그림 저작권 틀', + 'ta' :'Category:காப்புரிமை வார்ப்புருக்கள்', 'zh' : 'Category:版權申告模板', } @@ -436,6 +445,7 @@ 'de':'Benutzer:ABF/D3', 'en':None, 'ja':None, + 'ta':None, 'zh':None, } # Title of the email @@ -443,6 +453,7 @@ 'de':'Problemen mit Deinem Bild auf der Deutschen Wikipedia', 'en':None, 'ja':None, + 'ta':None, 'zh':None, }

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn April 2009