Pywikipedia-svn August 2010

pywikipedia-svn@lists.wikimedia.org

9 participants
99 discussions

SVN: [8393]
by xqt＠svn.wikimedia.org 09 Aug '10

09 Aug '10

Revision: 8393 Author: xqt Date: 2010-08-09 18:56:21 +0000 (Mon, 09 Aug 2010) Log Message: ----------- new Portal and Book namespaces for yo-wiki, update family files Modified Paths: -------------- branches/rewrite/pywikibot/families/wikipedia_family.py branches/rewrite/pywikibot/families/wikisource_family.py branches/rewrite/pywikibot/families/wiktionary_family.py trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/families/wikisource_family.py trunk/pywikipedia/families/wiktionary_family.py Modified: branches/rewrite/pywikibot/families/wikipedia_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikipedia_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ branches/rewrite/pywikibot/families/wikipedia_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -17,21 +17,21 @@ 'war', 'sl', 'hr', 'et', 'new', 'ms', 'simple', 'th', 'gl', 'roa-rup', 'nn', 'eu', 'hi', 'el', 'ht', 'te', 'la', 'ka', 'mk', 'ceb', 'az', 'br', 'sh', 'tl', 'mr', 'jv', 'lb', 'bs', 'lv', 'is', - 'cy', 'pms', 'sq', 'be-x-old', 'bpy', 'ta', 'oc', 'an', 'be', 'bn', + 'cy', 'pms', 'sq', 'be-x-old', 'bpy', 'ta', 'oc', 'be', 'an', 'bn', 'io', 'sw', 'fy', 'nds', 'lmo', 'scn', 'gu', 'af', 'qu', 'su', - 'zh-yue', 'ku', 'ur', 'ast', 'ml', 'nap', 'bat-smg', 'wa', 'cv', + 'zh-yue', 'ku', 'ur', 'ml', 'ast', 'nap', 'bat-smg', 'wa', 'cv', 'ga', 'ksh', 'hy', 'yo', 'tg', 'kn', 'roa-tara', 'vec', 'gd', 'yi', - 'uz', 'pam', 'zh-min-nan', 'os', 'tt', 'sah', 'als', 'ne', 'mi', + 'uz', 'pam', 'zh-min-nan', 'os', 'tt', 'sah', 'ne', 'als', 'mi', 'arz', 'kk', 'nah', 'li', 'hsb', 'glk', 'co', 'gan', 'pnb', 'ia', 'bcl', 'mn', 'fiu-vro', 'am', 'nds-nl', 'fo', 'vls', 'tk', 'sa', 'si', 'bar', 'dv', 'gv', 'nrm', 'pag', 'rm', 'map-bms', 'my', 'sco', 'diq', 'se', 'wuu', 'fur', 'lij', 'ug', 'bh', 'mt', 'nov', 'mzn', 'csb', 'mg', 'ilo', 'km', 'zh-classical', 'sc', 'lad', 'pi', 'ang', 'cbk-zam', 'frp', 'bo', 'ckb', 'hak', 'hif', 'pa', 'kw', 'xal', - 'szl', 'pdc', 'haw', 'ps', 'ie', 'stq', 'fj', 'crh', 'kv', 'to', - 'ace', 'myv', 'so', 'nv', 'gn', 'ky', 'krc', 'ln', 'ext', 'mhr', + 'szl', 'haw', 'pdc', 'ps', 'ie', 'stq', 'fj', 'crh', 'kv', 'to', + 'ace', 'myv', 'nv', 'so', 'gn', 'ky', 'krc', 'ln', 'ext', 'mhr', 'arc', 'jbo', 'wo', 'eml', 'ty', 'pcd', 'kab', 'tpi', 'pap', 'ay', - 'zea', 'ba', 'srn', 'udm', 'kl', 'ig', 'ce', 'or', 'dsb', 'ee', + 'zea', 'ba', 'srn', 'ee', 'kl', 'udm', 'ig', 'ce', 'or', 'dsb', 'kg', 'lo', 'ab', 'rmy', 'cu', 'mwl', 'mdf', 'kaa', 'sm', 'av', 'ks', 'tet', 'got', 'sd', 'na', 'bm', 'pih', 'pnt', 'iu', 'ik', 'as', 'cdo', 'bi', 'chr', 'bug', 'om', 'za', 'ss', 'ti', 'ts', 've', Modified: branches/rewrite/pywikibot/families/wikisource_family.py =================================================================== --- branches/rewrite/pywikibot/families/wikisource_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ branches/rewrite/pywikibot/families/wikisource_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -12,8 +12,8 @@ self.languages_by_size = [ 'en', 'zh', 'ru', 'pt', 'fr', 'de', 'es', 'it', 'he', 'ar', 'hu', - 'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'tr', 'nl', - 'sv', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi', + 'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'sv', 'nl', + 'tr', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi', 'ml', 'az', 'is', 'bs', 'hy', 'ca', 'id', 'mk', 'ta', 'da', 'no', 'et', 'bg', 'lt', 'gl', 'kn', 'cy', 'sk', 'zh-min-nan', 'fo', ] Modified: branches/rewrite/pywikibot/families/wiktionary_family.py =================================================================== --- branches/rewrite/pywikibot/families/wiktionary_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ branches/rewrite/pywikibot/families/wiktionary_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -14,17 +14,17 @@ 'en', 'fr', 'lt', 'zh', 'tr', 'ru', 'vi', 'io', 'pl', 'pt', 'fi', 'hu', 'el', 'ko', 'no', 'ta', 'de', 'sv', 'it', 'nl', 'kn', 'lo', 'ml', 'li', 'ja', 'es', 'ku', 'ar', 'et', 'ro', 'id', 'te', 'gl', - 'uk', 'bg', 'ca', 'vo', 'fa', 'is', 'scn', 'sr', 'simple', 'cs', - 'af', 'hr', 'th', 'oc', 'fy', 'sw', 'mg', 'br', 'he', 'la', 'sq', + 'uk', 'bg', 'ca', 'vo', 'fa', 'is', 'scn', 'mg', 'sr', 'simple', + 'cs', 'af', 'hr', 'th', 'oc', 'fy', 'sw', 'br', 'he', 'la', 'sq', 'sl', 'hy', 'tt', 'my', 'zh-min-nan', 'da', 'wa', 'ne', 'tk', 'ast', 'ur', 'hsb', 'cy', 'kk', 'ky', 'eo', 'ka', 'lv', 'wo', 'ang', 'co', 'hi', 'ga', 'az', 'gn', 'ia', 'eu', 'sk', 'ms', 'csb', 'tl', 'st', 'mr', 'nn', 'be', 'nds', 'kl', 'sd', 'ug', 'ps', 'ti', 'mk', 'sh', 'an', 'bn', 'gu', 'km', 'si', 'ss', 'ts', 'qu', 'bs', 'fo', 'am', - 'rw', 'chr', 'su', 'om', 'iu', 'mn', 'nah', 'yi', 'ie', 'gv', 'kw', - 'pa', 'mt', 'gd', 'tg', 'zu', 'dv', 'tpi', 'ik', 'sg', 'so', - 'roa-rup', 'za', 'lb', 'jv', 'uz', 'ln', 'mi', 'sm', 'ha', 'ay', - 'sa', 'na', 'jbo', 'tn', 'fj', 'ks', 'dz', 'als', + 'rw', 'chr', 'su', 'om', 'iu', 'mn', 'nah', 'kw', 'yi', 'ie', 'gv', + 'pa', 'mt', 'gd', 'tg', 'zu', 'dv', 'tpi', 'ik', 'sg', 'lb', 'so', + 'roa-rup', 'za', 'jv', 'uz', 'ln', 'mi', 'sm', 'ha', 'ay', 'sa', + 'na', 'jbo', 'tn', 'fj', 'ks', 'dz', 'als', ] for lang in self.languages_by_size: Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ trunk/pywikipedia/families/wikipedia_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -17,21 +17,21 @@ 'war', 'sl', 'hr', 'et', 'new', 'ms', 'simple', 'th', 'gl', 'roa-rup', 'nn', 'eu', 'hi', 'el', 'ht', 'te', 'la', 'ka', 'mk', 'ceb', 'az', 'br', 'sh', 'tl', 'mr', 'jv', 'lb', 'bs', 'lv', 'is', - 'cy', 'pms', 'sq', 'be-x-old', 'bpy', 'ta', 'oc', 'an', 'be', 'bn', + 'cy', 'pms', 'sq', 'be-x-old', 'bpy', 'ta', 'oc', 'be', 'an', 'bn', 'io', 'sw', 'fy', 'nds', 'lmo', 'scn', 'gu', 'af', 'qu', 'su', - 'zh-yue', 'ku', 'ur', 'ast', 'ml', 'nap', 'bat-smg', 'wa', 'cv', + 'zh-yue', 'ku', 'ur', 'ml', 'ast', 'nap', 'bat-smg', 'wa', 'cv', 'ga', 'ksh', 'hy', 'yo', 'tg', 'kn', 'roa-tara', 'vec', 'gd', 'yi', - 'uz', 'pam', 'zh-min-nan', 'os', 'tt', 'sah', 'als', 'ne', 'mi', + 'uz', 'pam', 'zh-min-nan', 'os', 'tt', 'sah', 'ne', 'als', 'mi', 'arz', 'kk', 'nah', 'li', 'hsb', 'glk', 'co', 'gan', 'pnb', 'ia', 'bcl', 'mn', 'fiu-vro', 'am', 'nds-nl', 'fo', 'vls', 'tk', 'sa', 'si', 'bar', 'dv', 'gv', 'nrm', 'pag', 'rm', 'map-bms', 'my', 'sco', 'diq', 'se', 'wuu', 'fur', 'lij', 'ug', 'bh', 'mt', 'nov', 'mzn', 'csb', 'mg', 'ilo', 'km', 'zh-classical', 'sc', 'lad', 'pi', 'ang', 'cbk-zam', 'frp', 'bo', 'ckb', 'hak', 'hif', 'pa', 'kw', 'xal', - 'szl', 'pdc', 'haw', 'ps', 'ie', 'stq', 'fj', 'crh', 'kv', 'to', - 'ace', 'myv', 'so', 'nv', 'gn', 'ky', 'krc', 'ln', 'ext', 'mhr', + 'szl', 'haw', 'pdc', 'ps', 'ie', 'stq', 'fj', 'crh', 'kv', 'to', + 'ace', 'myv', 'nv', 'so', 'gn', 'ky', 'krc', 'ln', 'ext', 'mhr', 'arc', 'jbo', 'wo', 'eml', 'ty', 'pcd', 'kab', 'tpi', 'pap', 'ay', - 'zea', 'ba', 'srn', 'udm', 'kl', 'ig', 'ce', 'or', 'dsb', 'ee', + 'zea', 'ba', 'srn', 'ee', 'kl', 'udm', 'ig', 'ce', 'or', 'dsb', 'kg', 'lo', 'ab', 'rmy', 'cu', 'mwl', 'mdf', 'kaa', 'sm', 'av', 'ks', 'tet', 'got', 'sd', 'na', 'bm', 'pih', 'pnt', 'iu', 'ik', 'as', 'cdo', 'bi', 'chr', 'bug', 'om', 'za', 'ss', 'ti', 'ts', 've', @@ -435,6 +435,7 @@ 'vi': u'Chủ đề', 'wuu': u'Transwiki', 'yi': u'פארטאל', + 'yo': u'Èbúté', 'zh': u'Portal', 'zh-classical': u'門', 'zh-min-nan': u'Portal', @@ -521,6 +522,7 @@ 'vi': u'Thảo luận Chủ đề', 'wuu': u'Transwiki talk', 'yi': u'פארטאל רעדן', + 'yo': u'Ọ̀rọ̀ èbúté', 'zh': u'Portal talk', 'zh-classical': u'議', 'zh-min-nam': u'Portal talk', @@ -603,11 +605,13 @@ self.namespaces[108] = { 'als': u'Spruch', 'en': u'Book', + 'yo': u'Ìwé', } self.namespaces[109] = { 'als': u'Spruch Diskussion', 'en': u'Book talk', + 'yo': u'Ọ̀rọ̀ ìwé', } self.namespaces[110] = { Modified: trunk/pywikipedia/families/wikisource_family.py =================================================================== --- trunk/pywikipedia/families/wikisource_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ trunk/pywikipedia/families/wikisource_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -12,8 +12,8 @@ self.languages_by_size = [ 'en', 'zh', 'ru', 'pt', 'fr', 'de', 'es', 'it', 'he', 'ar', 'hu', - 'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'tr', 'nl', - 'sv', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi', + 'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'sv', 'nl', + 'tr', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi', 'ml', 'az', 'is', 'bs', 'hy', 'ca', 'id', 'mk', 'ta', 'da', 'no', 'et', 'bg', 'lt', 'gl', 'kn', 'cy', 'sk', 'zh-min-nan', 'fo', ] Modified: trunk/pywikipedia/families/wiktionary_family.py =================================================================== --- trunk/pywikipedia/families/wiktionary_family.py 2010-08-09 04:21:27 UTC (rev 8392) +++ trunk/pywikipedia/families/wiktionary_family.py 2010-08-09 18:56:21 UTC (rev 8393) @@ -14,17 +14,17 @@ 'en', 'fr', 'lt', 'zh', 'tr', 'ru', 'vi', 'io', 'pl', 'pt', 'fi', 'hu', 'el', 'ko', 'no', 'ta', 'de', 'sv', 'it', 'nl', 'kn', 'lo', 'ml', 'li', 'ja', 'es', 'ku', 'ar', 'et', 'ro', 'id', 'te', 'gl', - 'uk', 'bg', 'ca', 'vo', 'fa', 'is', 'scn', 'sr', 'simple', 'cs', - 'af', 'hr', 'th', 'oc', 'fy', 'sw', 'mg', 'br', 'he', 'la', 'sq', + 'uk', 'bg', 'ca', 'vo', 'fa', 'is', 'scn', 'mg', 'sr', 'simple', + 'cs', 'af', 'hr', 'th', 'oc', 'fy', 'sw', 'br', 'he', 'la', 'sq', 'sl', 'hy', 'tt', 'my', 'zh-min-nan', 'da', 'wa', 'ne', 'tk', 'ast', 'ur', 'hsb', 'cy', 'kk', 'ky', 'eo', 'ka', 'lv', 'wo', 'ang', 'co', 'hi', 'ga', 'az', 'gn', 'ia', 'eu', 'sk', 'ms', 'csb', 'tl', 'st', 'mr', 'nn', 'be', 'nds', 'kl', 'sd', 'ug', 'ps', 'ti', 'mk', 'sh', 'an', 'bn', 'gu', 'km', 'si', 'ss', 'ts', 'qu', 'bs', 'fo', 'am', - 'rw', 'chr', 'su', 'om', 'iu', 'mn', 'nah', 'yi', 'ie', 'gv', 'kw', - 'pa', 'mt', 'gd', 'tg', 'zu', 'dv', 'tpi', 'ik', 'sg', 'so', - 'roa-rup', 'za', 'lb', 'jv', 'uz', 'ln', 'mi', 'sm', 'ha', 'ay', - 'sa', 'na', 'jbo', 'tn', 'fj', 'ks', 'dz', 'als', + 'rw', 'chr', 'su', 'om', 'iu', 'mn', 'nah', 'kw', 'yi', 'ie', 'gv', + 'pa', 'mt', 'gd', 'tg', 'zu', 'dv', 'tpi', 'ik', 'sg', 'lb', 'so', + 'roa-rup', 'za', 'jv', 'uz', 'ln', 'mi', 'sm', 'ha', 'ay', 'sa', + 'na', 'jbo', 'tn', 'fj', 'ks', 'dz', 'als', ] if family.config.SSL_connection:

1 0

SVN: [8392] trunk/pywikipedia/commonscat.py
by xqt＠svn.wikimedia.org 08 Aug '10

08 Aug '10

Revision: 8392 Author: xqt Date: 2010-08-09 04:21:27 +0000 (Mon, 09 Aug 2010) Log Message: ----------- Alias template has changes on de-wiki Modified Paths: -------------- trunk/pywikipedia/commonscat.py Modified: trunk/pywikipedia/commonscat.py =================================================================== --- trunk/pywikipedia/commonscat.py 2010-08-08 16:00:24 UTC (rev 8391) +++ trunk/pywikipedia/commonscat.py 2010-08-09 04:21:27 UTC (rev 8392) @@ -71,7 +71,7 @@ 'crh' : (u'CommonsKat', [u'Commonscat']), 'cs' : (u'Commonscat', [u'Commons cat']), 'da' : (u'Commonscat', [u'Commons cat', u'Commonskat', u'Commonscat2']), - 'de' : (u'Commonscat', [u'CommonsCat',]), + 'de' : (u'Commonscat', [u'Commons cat',]), 'en' : (u'Commons category', [u'Commoncat', u'Commons2', u'Cms-catlist-up', u'Catlst commons', u'Commonscategory', u'Commonscat', u'Commons cat']),

1 0

SVN: [8391] trunk/pywikipedia/imageuncat.py
by multichill＠svn.wikimedia.org 08 Aug '10

08 Aug '10

Revision: 8391 Author: multichill Date: 2010-08-08 16:00:24 +0000 (Sun, 08 Aug 2010) Log Message: ----------- Added {{Credit line}} and {{Own work}}. Modified Paths: -------------- trunk/pywikipedia/imageuncat.py Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2010-08-08 15:59:04 UTC (rev 8390) +++ trunk/pywikipedia/imageuncat.py 2010-08-08 16:00:24 UTC (rev 8391) @@ -253,6 +253,7 @@ u'Created by ForrestSjap', u'Created with Inkscape', u'Creator', + u'Credit line', u'Credits-panoramafotos.net', u'Crh', u'Crh-cyrl', @@ -609,6 +610,7 @@ u'Otrs pending', u'OTRS pending', u'Own', + u'Own work', u'Pa', u'Pag', u'PAGENAME',

1 0

SVN: [8390] trunk/pywikipedia/imagecopy_self.py
by multichill＠svn.wikimedia.org 08 Aug '10

08 Aug '10

Revision: 8390 Author: multichill Date: 2010-08-08 15:59:04 +0000 (Sun, 08 Aug 2010) Log Message: ----------- Now parsing {{Information}} with standard pywikipedia template parser functions. Should be much more flexible. Location field is now parsed and put at the start of the description field (same way as it's used at enwp). Modified Paths: -------------- trunk/pywikipedia/imagecopy_self.py Modified: trunk/pywikipedia/imagecopy_self.py =================================================================== --- trunk/pywikipedia/imagecopy_self.py 2010-08-08 14:58:29 UTC (rev 8389) +++ trunk/pywikipedia/imagecopy_self.py 2010-08-08 15:59:04 UTC (rev 8390) @@ -324,7 +324,7 @@ Build a new description based on the imagepage ''' if u'{{Information' in imagepage.get() or u'{{information' in imagepage.get(): - (description, date, source, author) = self.getNewFieldsFromInformation(imagepage) + (description, date, source, author, permission, other_versions) = self.getNewFieldsFromInformation(imagepage) else: (description, date, source, author) = self.getNewFieldsFromFreetext(imagepage) @@ -336,44 +336,69 @@ ''' Try to extract fields from the current information template for the new information template. ''' + + fields = [u'location', u'description', u'source', u'date', u'author', u'permission', u'other versions'] + description = u'' + source = u'' date = u'' - source = u'' author = u'' permission = u'' other_versions = u'' - text = imagepage.get() - # Need to add the permission field - # Need to use pywikipedia template parser code - regexes =[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', - u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', - ] - - for regex in regexes: - match =re.search(regex, text, re.IGNORECASE|re.DOTALL) - if match: - description = self.convertLinks(match.group(u'description').strip(), imagepage.site()) - - date = match.group(u'date').strip() - if date == u'': - date = self.getUploadDate(imagepage) - source = self.getSource(imagepage, source=self.convertLinks(match.group(u'source').strip(), imagepage.site())) + contents = {} - author = self.convertLinks(match.group(u'author').strip(), imagepage.site()) - if author == u'': - author = self.getAuthorText(imagepage) - - if u'permission' in match.groupdict(): - permission = self.convertLinks(match.group(u'permission').strip(), imagepage.site()) - if u'other_versions' in match.groupdict(): - other_versions = self.convertLinks(match.group(u'other_versions').strip(), imagepage.site()) - # Return the stuff we found - return (description, date, source, author) - - #We didn't find anything, return the empty strings - return (description, date, source, author) + for field in fields: + contents[field]=u'' + templates = imagepage.templatesWithParams() + + for (template, params) in templates: + if template==u'Information': + for param in params: + #Split at = + (field, sep, value) = param.partition(u'=') + #To lowercase, remove underscores and strip of spaces + field = field.lower().replace(u'_', u' ').strip() + #See if first part is in fields list + if field in fields: + #Ok, field is good, store it. + contents[field] = value.strip() + + # We now got the contents from the old information template. Let's get the info for the new one + + # Description + if not contents[u'location']==u'': + description = self.convertLinks(contents[u'location']) + u'\n' + if not contents[u'description']==u'': + description = description + self.convertLinks(contents[u'description'], imagepage.site()) + + # Source + source = self.getSource(imagepage, source=self.convertLinks(contents[u'source'], imagepage.site())) + + # Date + if not contents[u'date']==u'': + date = contents[u'date'] + else: + date = self.getUploadDate(imagepage) + + # Author + if not (contents[u'author']==u'' or contents[u'author']==self.getAuthor(imagepage)): + author = self.convertLinks(contents[u'author'], imagepage.site()) + else: + author = self.getAuthorText(imagepage) + + # Permission + # Still have to filter out crap like "see below" or "yes" + if not contents[u'permission']==u'': + permission = self.convertLinks(contents[u'permission'], imagepage.site()) + + # Other_versions + if not contents[u'other versions']==u'': + other_versions = self.convertLinks(contents[u'other versions'], imagepage.site()) + + return (description, date, source, author, permission, other_versions) + def getNewFieldsFromFreetext(self, imagepage): ''' Try to extract fields from free text for the new information template.

1 0

SVN: [8389] trunk/pywikipedia/imagecopy_self.py
by multichill＠svn.wikimedia.org 08 Aug '10

08 Aug '10

Revision: 8389 Author: multichill Date: 2010-08-08 14:58:29 +0000 (Sun, 08 Aug 2010) Log Message: ----------- Better name Added Paths: ----------- trunk/pywikipedia/imagecopy_self.py Copied: trunk/pywikipedia/imagecopy_self.py (from rev 8388, trunk/pywikipedia/imagecopy_enwp.py) =================================================================== --- trunk/pywikipedia/imagecopy_self.py (rev 0) +++ trunk/pywikipedia/imagecopy_self.py 2010-08-08 14:58:29 UTC (rev 8389) @@ -0,0 +1,721 @@ +# -*- coding: utf-8 -*- +""" +Script to copy self published files from the English Wikipedia to Wikimedia Commons. + +This bot is based on imagecopy.py and intended to be used to empty out http://en.wikipedia.org/wiki/Category:Self-published_work + +This bot uses a graphical interface and may not work from commandline +only environment. + +Examples + +Work on a single file + python imagecopy.py -page:file:<filename> +Work on all images in a category:<cat> + python imagecopy.py -cat:<cat> +Work on all images which transclude a template + python imagecopy.py -transcludes:<template> + +See pagegenerators.py for more ways to get a list of images. +By default the bot works on your home wiki (set in user-config) + +This is a first test version and should be used with care. + +Use -nochecktemplate if you don't want to add the check template. Be sure to check it yourself. + +Todo: +*Queues with threads have to be implemented for the information collecting part and for the upload part. +*Categories are now on a single line. Something like hotcat would be nice. + +""" +# +# Based on upload.py by: +# (C) Rob W.W. Hooft, Andre Engels 2003-2007 +# (C) Wikipedian, Keichwa, Leogregianin, Rikwade, Misza13 2003-2007 +# +# New bot by: +# (C) Kyle/Orgullomoore, Siebrand Mazeland 2007 +# +# Another rewrite by: +# (C) Multichill 2008 +# +# English Wikipedia specific bot by: +# (C) Multichill 2010 +# +# Distributed under the terms of the MIT license. +# +__version__='$Id$' +# + +from Tkinter import * +import os, sys, re, codecs +import urllib, httplib, urllib2 +import webbrowser +from Queue import Queue +import time, threading +import wikipedia, config, socket +import pagegenerators, add_text +import imagerecat +from datetime import datetime +from upload import * +from image import * +NL='' + +nowCommonsTemplate = { + 'en': u'{{NowCommons|1=File:%s|date=~~~~~|reviewer={{subst:REVISIONUSER}}}}', +} + +nowCommonsMessage = { + 'en': u'File is now available on Wikimedia Commons.', +} + +moveToCommonsTemplate = { + 'en': [u'Commons ok', u'Copy to Wikimedia Commons', u'Move to commons', u'Movetocommons', u'To commons', u'Copy to Wikimedia Commons by BotMultichill'], +} + +imageMoveMessage = { + 'en': u'[[:File:%s|File]] moved to [[:commons:File:%s|commons]].', +} + +skipTemplates = [u'Db-f1', + u'Db-f2', + u'Db-f3', + u'Db-f7', + u'Db-f8', + u'Db-f9', + u'Db-f10', + u'NowCommons', + u'CommonsNow', + u'Nowcommons', + u'NowCommonsThis', + u'Nowcommons2', + u'NCT', + u'Nowcommonsthis', + u'Moved to commons', + u'Now Commons', + u'Now at commons', + u'Db-nowcommons', + u'WikimediaCommons', + u'Now commons', + u'Di-no source', + u'Di-no license', + u'Di-no permission', + u'Di-orphaned fair use', + u'Di-no source no license', + u'Di-replaceable fair use', + u'Di-no fair use rationale', + u'Di-disputed fair use rationale', + u'Puf', + u'PUI', + u'Pui', + u'Ffd', + u'PD-user', # Only the self templates are supported for now. + ] + + +licenseTemplates = [(u'\{\{(self|self2)\|([^\}]+)\}\}', u'{{Self|\\2|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{(GFDL-self|GFDL-self-no-disclaimers)\|([^\}]+)\}\}', u'{{Self|GFDL|\\2|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{GFDL-self-with-disclaimers\|([^\}]+)\}\}', u'{{Self|GFDL-with-disclaimers|\\1|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{PD-self(\|date=[^\}]+)?\}\}', u'{{PD-user-w|%(lang)s|%(family)s|%(author)s}}'), + #Multilicense replacing placeholder + (u'\{\{Multilicense replacing placeholder new(\|class=[^\}]+)?\}\}', u'{{Self|GFDL|Cc-by-sa-3.0,2.5,2.0,1.0|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + ] + +sourceGarbage = [u'==\s*Summary\s*==', + u'==\s*Licensing:?\s*==', + u'\{\{(Copy to Wikimedia Commons|Move to Commons|Move to commons|Move to Wikimedia Commons|Copy to commons|Mtc|MtC|MTC|CWC|CtWC|CTWC|Ctwc|Tocommons|Copy to Commons|To Commons|Movetocommons|Move to Wikimedia commons|Move-to-commons|Commons ok|ToCommons|To commons|MoveToCommons|Copy to wikimedia commons|Upload to commons|CopyToCommons|Copytocommons|MITC|MovetoCommons|Do move to Commons)\}\}' + ] + +class Tkdialog: + def __init__(self, imagepage, description, date, source, author, licensetemplate, categories): + self.root=Tk() + #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) + #Always appear the same size and in the bottom-left corner + self.root.geometry("1000x400+100-100") + self.root.title(imagepage.titleWithoutNamespace()) + + + self.url=imagepage.permalink() + self.scrollbar=Scrollbar(self.root, orient=VERTICAL) + + self.old_description=Text(self.root) + self.old_description.insert(END, imagepage.get().encode('utf-8')) + self.old_description.config(state=DISABLED, height=8, width=140, padx=0, pady=0, wrap=WORD, yscrollcommand=self.scrollbar.set) + + self.scrollbar.config(command=self.old_description.yview) + + self.filename = imagepage.titleWithoutNamespace() + + self.description = description + self.date = date + self.source = source + self.author = author + self.licensetemplate = licensetemplate + self.categories = categories + self.skip = False + + self.old_description_label=Label(self.root,text=u'The old description was : ') + self.new_description_label=Label(self.root,text=u'The new fields are : ') + self.filename_label=Label(self.root,text=u'Filename : ') + self.information_description_label=Label(self.root,text=u'Description : ') + self.information_date_label=Label(self.root,text=u'Date : ') + self.information_source_label=Label(self.root,text=u'Source : ') + self.information_author_label=Label(self.root,text=u'Author : ') + self.information_licensetemplate_label=Label(self.root,text=u'License : ') + self.information_categories_label=Label(self.root,text=u'Categories : ') + + self.filename_field=Entry(self.root) + self.information_description=Entry(self.root) + self.information_date=Entry(self.root) + self.information_source=Entry(self.root) + self.information_author=Entry(self.root) + self.information_licensetemplate=Entry(self.root) + self.information_categories=Entry(self.root) + + self.field_width=120 + + self.filename_field.config(width=self.field_width) + self.information_description.config(width=self.field_width) + self.information_date.config(width=self.field_width) + self.information_source.config(width=self.field_width) + self.information_author.config(width=self.field_width) + self.information_licensetemplate.config(width=self.field_width) + self.information_categories.config(width=self.field_width) + + + self.filename_field.insert(0, self.filename) + self.information_description.insert(0, self.description) + self.information_date.insert(0, self.date) + self.information_source.insert(0, self.source) + self.information_author.insert(0, self.author) + self.information_licensetemplate.insert(0, self.licensetemplate) + self.information_categories.insert(0, self.categories) + + self.browserButton=Button(self.root, text='View in browser', command=self.openInBrowser) + self.skipButton=Button(self.root, text="Skip", command=self.skipFile) + self.okButton=Button(self.root, text="OK", command=self.okFile) + + ##Start grid + self.old_description_label.grid(row=0, column=0, columnspan=3) + + self.old_description.grid(row=1, column=0, columnspan=3) + self.scrollbar.grid(row=1, column=3) + self.new_description_label.grid(row=2, column=0, columnspan=3) + + self.filename_label.grid(row=3, column=0) + self.information_description_label.grid(row=4, column=0) + self.information_date_label.grid(row=5, column=0) + self.information_source_label.grid(row=6, column=0) + self.information_author_label.grid(row=7, column=0) + self.information_licensetemplate_label.grid(row=8, column=0) + self.information_categories_label.grid(row=9, column=0) + + self.filename_field.grid(row=3, column=1, columnspan=3) + self.information_description.grid(row=4, column=1, columnspan=3) + self.information_date.grid(row=5, column=1, columnspan=3) + self.information_source.grid(row=6, column=1, columnspan=3) + self.information_author.grid(row=7, column=1, columnspan=3) + self.information_licensetemplate.grid(row=8, column=1, columnspan=3) + self.information_categories.grid(row=9, column=1, columnspan=3) + + self.okButton.grid(row=10, column=3, rowspan=2) + self.skipButton.grid(row=10, column=2, rowspan=2) + self.browserButton.grid(row=10, column=1, rowspan=2) + + def okFile(self): + ''' + The user pressed the OK button. + ''' + self.filename=self.filename_field.get() + self.description=self.information_description.get() + self.date=self.information_date.get() + self.source=self.information_source.get() + self.author=self.information_author.get() + self.licensetemplate=self.information_licensetemplate.get() + self.categories=self.information_categories.get() + + self.root.destroy() + + def skipFile(self): + ''' + The user pressed the Skip button. + ''' + self.skip=1 + self.root.destroy() + + def openInBrowser(self): + ''' + The user pressed the View in browser button. + ''' + webbrowser.open(self.url) + + def add2autoskip(self): + ''' + The user pressed the Add to AutoSkip button. + ''' + templateid=int(self.templatelist.curselection()[0]) + template=self.templatelist.get(templateid) + toadd=codecs.open(archivo, 'a', 'utf-8') + toadd.write('{{'+template) + toadd.close() + self.skipFile() + + def getnewmetadata(self): + ''' + Activate the dialog and return the new name and if the image is skipped. + ''' + self.root.mainloop() + return (self.filename, self.description, self.date, self.source, self.author, self.licensetemplate, self.categories, self.skip) + + +class imageFetcher(threading.Thread): + ''' + Tries to fetch information for all images in the generator + ''' + def __init__ ( self, pagegenerator, prefetchQueue): + self.pagegenerator = pagegenerator + self.prefetchQueue = prefetchQueue + imagerecat.initLists() + threading.Thread.__init__ ( self ) + + def run(self): + for page in self.pagegenerator: + self.processImage(page) + self.prefetchQueue.put(None) + wikipedia.output(u'Fetched all images.') + return True + + def processImage(self, page): + ''' + Work on a single image + ''' + if page.exists() and (page.namespace() == 6) and (not page.isRedirectPage()): + imagepage = wikipedia.ImagePage(page.site(), page.title()) + + #First do autoskip. + if self.doiskip(imagepage): + wikipedia.output(u'Skipping %s : Got a template on the skip list.' % page.title()) + return False + + text = imagepage.get() + foundMatch = False + for (regex, replacement) in licenseTemplates: + match = re.search(regex, text, re.IGNORECASE) + if match: + foundMatch = True + if not foundMatch: + wikipedia.output(u'Skipping %s : No suitable license template was found.' % page.title()) + return False + self.prefetchQueue.put(self.getNewFields(imagepage)) + + def doiskip(self, imagepage): + ''' + Skip this image or not. + Returns True if the image is on the skip list, otherwise False + ''' + for template in imagepage.templates(): + if template in skipTemplates: + wikipedia.output(u'Found ' + template + u' which is on the template skip list') + return True + return False + + def getNewFields(self, imagepage): + ''' + Build a new description based on the imagepage + ''' + if u'{{Information' in imagepage.get() or u'{{information' in imagepage.get(): + (description, date, source, author) = self.getNewFieldsFromInformation(imagepage) + else: + (description, date, source, author) = self.getNewFieldsFromFreetext(imagepage) + + licensetemplate = self.getNewLicensetemplate(imagepage) + categories = self.getNewCategories(imagepage) + return (imagepage, description, date, source, author, licensetemplate, categories) + + def getNewFieldsFromInformation(self, imagepage): + ''' + Try to extract fields from the current information template for the new information template. + ''' + description = u'' + date = u'' + source = u'' + author = u'' + permission = u'' + other_versions = u'' + text = imagepage.get() + # Need to add the permission field + # Need to use pywikipedia template parser code + regexes =[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', + u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', + ] + + for regex in regexes: + match =re.search(regex, text, re.IGNORECASE|re.DOTALL) + if match: + description = self.convertLinks(match.group(u'description').strip(), imagepage.site()) + + date = match.group(u'date').strip() + if date == u'': + date = self.getUploadDate(imagepage) + + source = self.getSource(imagepage, source=self.convertLinks(match.group(u'source').strip(), imagepage.site())) + + author = self.convertLinks(match.group(u'author').strip(), imagepage.site()) + if author == u'': + author = self.getAuthorText(imagepage) + + if u'permission' in match.groupdict(): + permission = self.convertLinks(match.group(u'permission').strip(), imagepage.site()) + if u'other_versions' in match.groupdict(): + other_versions = self.convertLinks(match.group(u'other_versions').strip(), imagepage.site()) + # Return the stuff we found + return (description, date, source, author) + + #We didn't find anything, return the empty strings + return (description, date, source, author) + + def getNewFieldsFromFreetext(self, imagepage): + ''' + Try to extract fields from free text for the new information template. + ''' + text = imagepage.get() + #text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE) + #text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE) + #text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text, re.IGNORECASE) + + for toRemove in sourceGarbage: + text = re.sub(toRemove, u'', text, re.IGNORECASE) + + for (regex, repl) in licenseTemplates: + text = re.sub(regex, u'', text, re.IGNORECASE) + + text = wikipedia.removeCategoryLinks(text, imagepage.site()).strip() + + description = self.convertLinks(text.strip(), imagepage.site()) + date = self.getUploadDate(imagepage) + source = self.getSource(imagepage) + author = self.getAuthorText(imagepage) + return (description, date, source, author) + + def getUploadDate(self, imagepage): + ''' + Get the original upload date to put in the date field of the new information template. If we really have nothing better. + ''' + uploadtime = imagepage.getFileVersionHistory()[-1][0] + uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ') + return u'{{Date|' + str(uploadDatetime.year) + u'|' + str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + u'}} (original upload date)' + + def getSource(self, imagepage, source=u''): + ''' + Get the text to put in the source field of the new information template. + ''' + site = imagepage.site() + lang = site.language() + family = site.family.name + if source==u'': + source=u'{{Own}}' + + return source.strip() + u'<BR />Transferred from [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family} + + def getAuthorText(self, imagepage): + ''' + Get the original uploader to put in the author field of the new information template. + ''' + site = imagepage.site() + lang = site.language() + family = site.family.name + + firstuploader = self.getAuthor(imagepage) + return u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family , u'firstuploader' : firstuploader} + + def getAuthor(self, imagepage): + ''' + Get the first uploader. + ''' + return imagepage.getFileVersionHistory()[-1][1].strip() + + def convertLinks(self, text, sourceSite): + ''' + Convert links from the current wiki to Commons. + ''' + lang = sourceSite.language() + family = sourceSite.family.name + conversions =[(u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\2]]'), + (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'), + ] + + for (regex, replacement) in conversions: + text = re.sub(regex, replacement % {u'lang' : lang, u'family' : family}, text) + + return text + + def getNewLicensetemplate(self, imagepage): + ''' + Get a license template to put on the image to be uploaded + ''' + text = imagepage.get() + + site = imagepage.site() + lang = site.language() + family = site.family.name + + result = u'' + + for (regex, replacement) in licenseTemplates: + match = re.search(regex, text, re.IGNORECASE) + if match: + result = re.sub(regex, replacement, match.group(0), re.IGNORECASE) + return result % {u'author' : self.getAuthor(imagepage), + u'lang' : lang, + u'family' : family} + + return result + + def getNewCategories(self, imagepage): + ''' + Get a categories for the image + Dont forget to filter + ''' + result = u'' + (commonshelperCats, usage, galleries) = imagerecat.getCommonshelperCats(imagepage) + newcats = imagerecat.applyAllFilters(commonshelperCats) + for newcat in newcats: + result = result + u'[[Category:' + newcat + u']] ' + return result + +class userInteraction(threading.Thread): + ''' + Prompt all images to the user. + ''' + def __init__ ( self, prefetchQueue, uploadQueue): + self.prefetchQueue = prefetchQueue + self.uploadQueue = uploadQueue + threading.Thread.__init__ ( self ) + + def run(self): + while True: + fields = self.prefetchQueue.get() + if fields: + self.processImage(fields) + else: + break + self.uploadQueue.put(None) + wikipedia.output(u'User worked on all images.') + return True + + def processImage(self, fields): + ''' + Work on a single image + ''' + (imagepage, description, date, source, author, licensetemplate, categories) = fields + while True: + # Do the Tkdialog to accept/reject and change te name + (filename, description, date, source, author, licensetemplate, categories, skip)=Tkdialog(imagepage, description, date, source, author, licensetemplate, categories).getnewmetadata() + + if skip: + wikipedia.output(u'Skipping %s : User pressed skip.' % imagepage.title()) + return False + + # Check if the image already exists + CommonsPage=wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename) + if not CommonsPage.exists(): + break + else: + wikipedia.output('Image already exists, pick another name or skip this image') + # We dont overwrite images, pick another name, go to the start of the loop + + self.uploadQueue.put((imagepage, filename, description, date, source, author, licensetemplate, categories)) + + +class uploader(threading.Thread): + ''' + Upload all images + ''' + def __init__ ( self, uploadQueue): + self.uploadQueue = uploadQueue + self.checktemplate = True + threading.Thread.__init__ ( self ) + + def run(self): + while True: #Change later + fields = self.uploadQueue.get() + if fields: + self.processImage(fields) + else: + break + return True + + def nochecktemplate(self): + ''' + Don't want to add {{BotMoveToCommons}} + ''' + self.checktemplate = False + return + + def processImage(self, fields): + ''' + Work on a single image + ''' + (imagepage, filename, description, date, source, author, licensetemplate, categories) = fields + cid = self.buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories) + wikipedia.output(cid) + bot = UploadRobot(url=imagepage.fileUrl(), description=cid, useFilename=filename, keepFilename=True, verifyDescription=False, ignoreWarning = True, targetSite = wikipedia.getSite('commons', 'commons')) + bot.run() + + self.tagNowcommons(imagepage, filename) + self.replaceUsage(imagepage, filename) + + + def buildNewImageDescription(self, imagepage, description, date, source, author, licensetemplate, categories): + ''' + Build a new information template + ''' + + site = imagepage.site() + lang = site.language() + family = site.family.name + + cid = u'' + if self.checktemplate: + cid = cid + u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n' % {u'lang' : lang, u'family' : family} + cid = cid + u'== {{int:filedesc}} ==\n' + cid = cid + u'{{Information\n' + cid = cid + u'|description={{%(lang)s|1=' % {u'lang' : lang, u'family' : family} + cid = cid + description + u'}}\n' + cid = cid + u'|date=' + date + u'\n' + cid = cid + u'|source=' + source + u'\n' + cid = cid + u'|author=' + author + u'\n' + cid = cid + u'|permission=\n' + cid = cid + u'|other_versions=\n' + cid = cid + u'}}\n' + cid = cid + u'== {{int:license}} ==\n' + cid = cid + licensetemplate + u'\n' + cid = cid + u'\n' + cid = cid + self.getOriginalUploadLog(imagepage) + cid = cid + u'__NOTOC__\n' + if categories.strip()==u'': + cid = cid + u'{{Subst:Unc}}' + else: + cid = cid + categories + return cid + + def getOriginalUploadLog(self, imagepage): + ''' + Get the original upload log to put at the bottom of the image description page at Commons. + ''' + filehistory = imagepage.getFileVersionHistory() + filehistory.reverse() + + site = imagepage.site() + lang = site.language() + family = site.family.name + + sourceimage = imagepage.site().get_address(imagepage.title()).replace(u'&redirect=no&useskin=monobook', u'') + + result = u'== {{Original upload log}} ==\n' + result = result + u'The original description page is/was [http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following user names refer to %(lang)s.%(family)s.\n' % {u'lang' : lang, u'family' : family , u'sourceimage' : sourceimage} + for (timestamp, username, resolution, size, comment) in filehistory: + date = datetime.strptime(timestamp, u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M') + result = result + u'* %(date)s [[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s (%(size)s bytes) \'\'<nowiki>%(comment)s</nowiki>\'\'\n' % { + u'lang' : lang, + u'family' : family , + u'date' : date, + u'username' : username, + u'resolution': resolution, + u'size': size, + u'comment' : comment} + + return result + + def tagNowcommons(self, imagepage, filename): + ''' + Tagged the imag which has been moved to Commons for deletion. + ''' + if wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename).exists(): + #Get a fresh copy, force to get the page so we dont run into edit conflicts + imtxt=imagepage.get(force=True) + + #Remove the move to commons templates + if imagepage.site().language() in moveToCommonsTemplate: + for moveTemplate in moveToCommonsTemplate[imagepage.site().language()]: + imtxt = re.sub(u'(?i)\{\{' + moveTemplate + u'[^\}]*\}\}', u'', imtxt) + + #add {{NowCommons}} + if imagepage.site().language() in nowCommonsTemplate: + addTemplate = nowCommonsTemplate[imagepage.site().language()] % filename + else: + addTemplate = nowCommonsTemplate['_default'] % filename + + if imagepage.site().language() in nowCommonsMessage: + commentText = nowCommonsMessage[imagepage.site().language()] + else: + commentText = nowCommonsMessage['_default'] + + wikipedia.showDiff(imagepage.get(), imtxt + addTemplate) + imagepage.put(imtxt + addTemplate, comment = commentText) + + def replaceUsage(self, imagepage, filename): + ''' + If the image is uploaded under a different name, replace all usage. + ''' + if imagepage.titleWithoutNamespace() != filename: + gen = pagegenerators.FileLinksGenerator(imagepage) + preloadingGen = pagegenerators.PreloadingGenerator(gen) + + if imagepage.site().language() in imageMoveMessage: + moveSummary = imageMoveMessage[imagepage.site().language()] % (imagepage.titleWithoutNamespace(), filename) + else: + moveSummary = imageMoveMessage['_default'] % (imagepage.titleWithoutNamespace(), filename) + imagebot = ImageRobot(generator = preloadingGen, oldImage = imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary, always = True, loose = True) + imagebot.run() + + +def main(args): + wikipedia.output(u'WARNING: This is an experimental bot') + wikipedia.output(u'WARNING: It will only work on self published work images') + wikipedia.output(u'WARNING: This bot is still full of bugs') + wikipedia.output(u'WARNING: Use at your own risk!') + + generator = None; + always = False + checkTemplate = True + + # Load a lot of default generators + genFactory = pagegenerators.GeneratorFactory() + + for arg in wikipedia.handleArgs(): + if arg == '-nochecktemplate': + checkTemplate = False + else: + genFactory.handleArg(arg) + + generator = genFactory.getCombinedGenerator() + if not generator: + raise add_text.NoEnoughData('You have to specify the generator you want to use for the script!') + + pregenerator = pagegenerators.PreloadingGenerator(generator) + + prefetchQueue = Queue(maxsize=50) + uploadQueue = Queue(maxsize=200) + + imageFetcherThread = imageFetcher(pregenerator, prefetchQueue) + userInteractionThread = userInteraction(prefetchQueue, uploadQueue) + uploaderThread = uploader(uploadQueue) + + imageFetcherThread.daemon=False + userInteractionThread.daemon=False + uploaderThread.daemon=False + + if not checkTemplate: + uploaderThread.nochecktemplate() + + fetchDone = imageFetcherThread.start() + userDone = userInteractionThread.start() + uploadDone = uploaderThread.start() + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + finally: + wikipedia.stopme()

1 0

SVN: [8388] trunk/pywikipedia/imagecopy_enwp.py
by multichill＠svn.wikimedia.org 07 Aug '10

07 Aug '10

Revision: 8388 Author: multichill Date: 2010-08-07 16:08:55 +0000 (Sat, 07 Aug 2010) Log Message: ----------- Implemented 3 different threads 1. Fetcher thread gets all information 2. User thread does the user interaction 3. Uploader thread does the actual uploading This speeds up the bot a lot. Modified Paths: -------------- trunk/pywikipedia/imagecopy_enwp.py Modified: trunk/pywikipedia/imagecopy_enwp.py =================================================================== --- trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 13:37:23 UTC (rev 8387) +++ trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 16:08:55 UTC (rev 8388) @@ -51,6 +51,7 @@ import os, sys, re, codecs import urllib, httplib, urllib2 import webbrowser +from Queue import Queue import time, threading import wikipedia, config, socket import pagegenerators, add_text @@ -267,255 +268,252 @@ return (self.filename, self.description, self.date, self.source, self.author, self.licensetemplate, self.categories, self.skip) -def doiskip(imagepage): +class imageFetcher(threading.Thread): ''' - Skip this image or not. - Returns True if the image is on the skip list, otherwise False + Tries to fetch information for all images in the generator ''' - for template in imagepage.templates(): - if template in skipTemplates: - wikipedia.output(u'Found ' + template + u' which is on the template skip list') - return True - return False + def __init__ ( self, pagegenerator, prefetchQueue): + self.pagegenerator = pagegenerator + self.prefetchQueue = prefetchQueue + imagerecat.initLists() + threading.Thread.__init__ ( self ) -def getNewFields(imagepage): - ''' - Build a new description based on the imagepage - ''' - if u'{{Information' in imagepage.get() or u'{{information' in imagepage.get(): - (description, date, source, author) = getNewFieldsFromInformation(imagepage) - else: - (description, date, source, author) = getNewFieldsFromFreetext(imagepage) + def run(self): + for page in self.pagegenerator: + self.processImage(page) + self.prefetchQueue.put(None) + wikipedia.output(u'Fetched all images.') + return True - licensetemplate = getNewLicensetemplate(imagepage) - categories = getNewCategories(imagepage) - return (description, date, source, author, licensetemplate, categories) + def processImage(self, page): + ''' + Work on a single image + ''' + if page.exists() and (page.namespace() == 6) and (not page.isRedirectPage()): + imagepage = wikipedia.ImagePage(page.site(), page.title()) -def getNewFieldsFromInformation(imagepage): - ''' - ''' - description = u'' - date = u'' - source = u'' - author = u'' - permission = u'' - other_versions = u'' - text = imagepage.get() - # Need to add the permission field - # Need to use pywikipedia template parser code - regexes =[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', - u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', - ] + #First do autoskip. + if self.doiskip(imagepage): + wikipedia.output(u'Skipping %s : Got a template on the skip list.' % page.title()) + return False + text = imagepage.get() + foundMatch = False + for (regex, replacement) in licenseTemplates: + match = re.search(regex, text, re.IGNORECASE) + if match: + foundMatch = True + if not foundMatch: + wikipedia.output(u'Skipping %s : No suitable license template was found.' % page.title()) + return False + self.prefetchQueue.put(self.getNewFields(imagepage)) - for regex in regexes: - match =re.search(regex, text, re.IGNORECASE|re.DOTALL) - if match: - description = convertLinks(match.group(u'description').strip(), imagepage.site()) - - date = match.group(u'date').strip() - if date == u'': - date = getUploadDate(imagepage) + def doiskip(self, imagepage): + ''' + Skip this image or not. + Returns True if the image is on the skip list, otherwise False + ''' + for template in imagepage.templates(): + if template in skipTemplates: + wikipedia.output(u'Found ' + template + u' which is on the template skip list') + return True + return False - source = getSource(imagepage, source=convertLinks(match.group(u'source').strip(), imagepage.site())) + def getNewFields(self, imagepage): + ''' + Build a new description based on the imagepage + ''' + if u'{{Information' in imagepage.get() or u'{{information' in imagepage.get(): + (description, date, source, author) = self.getNewFieldsFromInformation(imagepage) + else: + (description, date, source, author) = self.getNewFieldsFromFreetext(imagepage) - author = convertLinks(match.group(u'author').strip(), imagepage.site()) - if author == u'': - author = getAuthorText(imagepage) - - if u'permission' in match.groupdict(): - permission = convertLinks(match.group(u'permission').strip(), imagepage.site()) - if u'other_versions' in match.groupdict(): - other_versions = convertLinks(match.group(u'other_versions').strip(), imagepage.site()) - # Return the stuff we found - return (description, date, source, author) - - #We didn't find anything, return the empty strings - return (description, date, source, author) + licensetemplate = self.getNewLicensetemplate(imagepage) + categories = self.getNewCategories(imagepage) + return (imagepage, description, date, source, author, licensetemplate, categories) -def getNewFieldsFromFreetext(imagepage): - ''' - ''' - text = imagepage.get() - #text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE) - #text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE) - #text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text, re.IGNORECASE) + def getNewFieldsFromInformation(self, imagepage): + ''' + Try to extract fields from the current information template for the new information template. + ''' + description = u'' + date = u'' + source = u'' + author = u'' + permission = u'' + other_versions = u'' + text = imagepage.get() + # Need to add the permission field + # Need to use pywikipedia template parser code + regexes =[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', + u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', + ] + + for regex in regexes: + match =re.search(regex, text, re.IGNORECASE|re.DOTALL) + if match: + description = self.convertLinks(match.group(u'description').strip(), imagepage.site()) + + date = match.group(u'date').strip() + if date == u'': + date = self.getUploadDate(imagepage) - for toRemove in sourceGarbage: - text = re.sub(toRemove, u'', text, re.IGNORECASE) - - for (regex, repl) in licenseTemplates: - text = re.sub(regex, u'', text, re.IGNORECASE) + source = self.getSource(imagepage, source=self.convertLinks(match.group(u'source').strip(), imagepage.site())) - text = wikipedia.removeCategoryLinks(text, imagepage.site()).strip() + author = self.convertLinks(match.group(u'author').strip(), imagepage.site()) + if author == u'': + author = self.getAuthorText(imagepage) + + if u'permission' in match.groupdict(): + permission = self.convertLinks(match.group(u'permission').strip(), imagepage.site()) + if u'other_versions' in match.groupdict(): + other_versions = self.convertLinks(match.group(u'other_versions').strip(), imagepage.site()) + # Return the stuff we found + return (description, date, source, author) - description = convertLinks(text.strip(), imagepage.site()) - date = getUploadDate(imagepage) - source = getSource(imagepage) - author = getAuthorText(imagepage) - return (description, date, source, author) + #We didn't find anything, return the empty strings + return (description, date, source, author) -def getUploadDate(imagepage): - # Get the original upload date - uploadtime = imagepage.getFileVersionHistory()[-1][0] - uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ') - return u'{{Date|' + str(uploadDatetime.year) + u'|' + str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + u'}} (original upload date)' + def getNewFieldsFromFreetext(self, imagepage): + ''' + Try to extract fields from free text for the new information template. + ''' + text = imagepage.get() + #text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE) + #text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE) + #text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text, re.IGNORECASE) -def getSource(imagepage, source=u''): - site = imagepage.site() - lang = site.language() - family = site.family.name - if source==u'': - source=u'{{Own}}' + for toRemove in sourceGarbage: + text = re.sub(toRemove, u'', text, re.IGNORECASE) - return source.strip() + u'<BR />Transferred from [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family} + for (regex, repl) in licenseTemplates: + text = re.sub(regex, u'', text, re.IGNORECASE) -def getAuthorText(imagepage): - site = imagepage.site() - lang = site.language() - family = site.family.name - - firstuploader = getAuthor(imagepage) - #FIXME : Make other sites than Wikipedia work - return u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family , u'firstuploader' : firstuploader} + text = wikipedia.removeCategoryLinks(text, imagepage.site()).strip() + + description = self.convertLinks(text.strip(), imagepage.site()) + date = self.getUploadDate(imagepage) + source = self.getSource(imagepage) + author = self.getAuthorText(imagepage) + return (description, date, source, author) -def getAuthor(imagepage): - return imagepage.getFileVersionHistory()[-1][1].strip() + def getUploadDate(self, imagepage): + ''' + Get the original upload date to put in the date field of the new information template. If we really have nothing better. + ''' + uploadtime = imagepage.getFileVersionHistory()[-1][0] + uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ') + return u'{{Date|' + str(uploadDatetime.year) + u'|' + str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + u'}} (original upload date)' -def convertLinks(text, sourceSite): - lang = sourceSite.language() - family = sourceSite.family.name - conversions =[(u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\2]]'), - (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'), - ] - - for (regex, replacement) in conversions: - text = re.sub(regex, replacement % {u'lang' : lang, u'family' : family}, text) + def getSource(self, imagepage, source=u''): + ''' + Get the text to put in the source field of the new information template. + ''' + site = imagepage.site() + lang = site.language() + family = site.family.name + if source==u'': + source=u'{{Own}}' + + return source.strip() + u'<BR />Transferred from [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family} - return text + def getAuthorText(self, imagepage): + ''' + Get the original uploader to put in the author field of the new information template. + ''' + site = imagepage.site() + lang = site.language() + family = site.family.name + + firstuploader = self.getAuthor(imagepage) + return u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang, u'family' : family , u'firstuploader' : firstuploader} -def getNewLicensetemplate(imagepage): - ''' - ''' - text = imagepage.get() - - site = imagepage.site() - lang = site.language() - family = site.family.name + def getAuthor(self, imagepage): + ''' + Get the first uploader. + ''' + return imagepage.getFileVersionHistory()[-1][1].strip() - result = u'' - - for (regex, replacement) in licenseTemplates: - match = re.search(regex, text, re.IGNORECASE) - if match: - result = re.sub(regex, replacement, match.group(0), re.IGNORECASE) - return result % {u'author' : getAuthor(imagepage), - u'lang' : lang, - u'family' : family} + def convertLinks(self, text, sourceSite): + ''' + Convert links from the current wiki to Commons. + ''' + lang = sourceSite.language() + family = sourceSite.family.name + conversions =[(u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\2]]'), + (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'), + ] - return result - -def getNewCategories(imagepage): - ''' - Get a categories for the image - Dont forget to filter - ''' - result = u'' - (commonshelperCats, usage, galleries) = imagerecat.getCommonshelperCats(imagepage) - newcats = imagerecat.applyAllFilters(commonshelperCats) - for newcat in newcats: - result = result + u'[[Category:' + newcat + u']] ' - return result + for (regex, replacement) in conversions: + text = re.sub(regex, replacement % {u'lang' : lang, u'family' : family}, text) -def getOriginalUploadLog(imagepage): - filehistory = imagepage.getFileVersionHistory() - filehistory.reverse() + return text - site = imagepage.site() - lang = site.language() - family = site.family.name - - sourceimage = imagepage.site().get_address(imagepage.title()).replace(u'&redirect=no&useskin=monobook', u'') - - result = u'== {{Original upload log}} ==\n' - result = result + u'The original description page is/was [http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following user names refer to %(lang)s.%(family)s.\n' % {u'lang' : lang, u'family' : family , u'sourceimage' : sourceimage} - for (timestamp, username, resolution, size, comment) in filehistory: - date = datetime.strptime(timestamp, u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M') - result = result + u'* %(date)s [[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s (%(size)s bytes) \'\'<nowiki>%(comment)s</nowiki>\'\'\n' % { - u'lang' : lang, - u'family' : family , - u'date' : date, - u'username' : username, - u'resolution': resolution, - u'size': size, - u'comment' : comment} + def getNewLicensetemplate(self, imagepage): + ''' + Get a license template to put on the image to be uploaded + ''' + text = imagepage.get() - return result + site = imagepage.site() + lang = site.language() + family = site.family.name - + result = u'' -def buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories, checkTemplate): - ''' - Build a new information template - ''' - - site = imagepage.site() - lang = site.language() - family = site.family.name - - cid = u'' - if checkTemplate: - cid = cid + u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n' % {u'lang' : lang, u'family' : family} - cid = cid + u'== {{int:filedesc}} ==\n' - cid = cid + u'{{Information\n' - cid = cid + u'|description={{%(lang)s|1=' % {u'lang' : lang, u'family' : family} - cid = cid + description + u'}}\n' - cid = cid + u'|date=' + date + u'\n' - cid = cid + u'|source=' + source + u'\n' - cid = cid + u'|author=' + author + u'\n' - cid = cid + u'|permission=\n' - cid = cid + u'|other_versions=\n' - cid = cid + u'}}\n' - cid = cid + u'== {{int:license}} ==\n' - cid = cid + licensetemplate + u'\n' - cid = cid + u'\n' - cid = cid + getOriginalUploadLog(imagepage) - cid = cid + u'__NOTOC__\n' - if categories.strip()==u'': - cid = cid + u'{{Subst:Unc}}' - else: - cid = cid + categories - return cid - - -def processImage(page, checkTemplate): - skip = False - if page.exists() and (page.namespace() == 6) and (not page.isRedirectPage()): - imagepage = wikipedia.ImagePage(page.site(), page.title()) - - #First do autoskip. - if doiskip(imagepage): - wikipedia.output(u'Skipping %s : Got a template on the skip list.' % page.title()) - return False - - text = imagepage.get() - foundMatch = False for (regex, replacement) in licenseTemplates: match = re.search(regex, text, re.IGNORECASE) if match: - foundMatch = True - if not foundMatch: - wikipedia.output(u'Skipping %s : No suitable license template was found.' % page.title()) - return False + result = re.sub(regex, replacement, match.group(0), re.IGNORECASE) + return result % {u'author' : self.getAuthor(imagepage), + u'lang' : lang, + u'family' : family} + + return result - (description, date, source, author, licensetemplate, categories) = getNewFields(imagepage) + def getNewCategories(self, imagepage): + ''' + Get a categories for the image + Dont forget to filter + ''' + result = u'' + (commonshelperCats, usage, galleries) = imagerecat.getCommonshelperCats(imagepage) + newcats = imagerecat.applyAllFilters(commonshelperCats) + for newcat in newcats: + result = result + u'[[Category:' + newcat + u']] ' + return result +class userInteraction(threading.Thread): + ''' + Prompt all images to the user. + ''' + def __init__ ( self, prefetchQueue, uploadQueue): + self.prefetchQueue = prefetchQueue + self.uploadQueue = uploadQueue + threading.Thread.__init__ ( self ) + + def run(self): while True: + fields = self.prefetchQueue.get() + if fields: + self.processImage(fields) + else: + break + self.uploadQueue.put(None) + wikipedia.output(u'User worked on all images.') + return True + + def processImage(self, fields): + ''' + Work on a single image + ''' + (imagepage, description, date, source, author, licensetemplate, categories) = fields + while True: # Do the Tkdialog to accept/reject and change te name (filename, description, date, source, author, licensetemplate, categories, skip)=Tkdialog(imagepage, description, date, source, author, licensetemplate, categories).getnewmetadata() if skip: - wikipedia.output(u'Skipping %s : User pressed skip.' % page.title()) + wikipedia.output(u'Skipping %s : User pressed skip.' % imagepage.title()) return False # Check if the image already exists @@ -524,13 +522,115 @@ break else: wikipedia.output('Image already exists, pick another name or skip this image') - # We dont overwrite images, pick another name, go to the start of the loop - - cid = buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories, checkTemplate) + # We dont overwrite images, pick another name, go to the start of the loop + + self.uploadQueue.put((imagepage, filename, description, date, source, author, licensetemplate, categories)) + + +class uploader(threading.Thread): + ''' + Upload all images + ''' + def __init__ ( self, uploadQueue): + self.uploadQueue = uploadQueue + self.checktemplate = True + threading.Thread.__init__ ( self ) + + def run(self): + while True: #Change later + fields = self.uploadQueue.get() + if fields: + self.processImage(fields) + else: + break + return True + + def nochecktemplate(self): + ''' + Don't want to add {{BotMoveToCommons}} + ''' + self.checktemplate = False + return + + def processImage(self, fields): + ''' + Work on a single image + ''' + (imagepage, filename, description, date, source, author, licensetemplate, categories) = fields + cid = self.buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories) wikipedia.output(cid) bot = UploadRobot(url=imagepage.fileUrl(), description=cid, useFilename=filename, keepFilename=True, verifyDescription=False, ignoreWarning = True, targetSite = wikipedia.getSite('commons', 'commons')) bot.run() + self.tagNowcommons(imagepage, filename) + self.replaceUsage(imagepage, filename) + + + def buildNewImageDescription(self, imagepage, description, date, source, author, licensetemplate, categories): + ''' + Build a new information template + ''' + + site = imagepage.site() + lang = site.language() + family = site.family.name + + cid = u'' + if self.checktemplate: + cid = cid + u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n' % {u'lang' : lang, u'family' : family} + cid = cid + u'== {{int:filedesc}} ==\n' + cid = cid + u'{{Information\n' + cid = cid + u'|description={{%(lang)s|1=' % {u'lang' : lang, u'family' : family} + cid = cid + description + u'}}\n' + cid = cid + u'|date=' + date + u'\n' + cid = cid + u'|source=' + source + u'\n' + cid = cid + u'|author=' + author + u'\n' + cid = cid + u'|permission=\n' + cid = cid + u'|other_versions=\n' + cid = cid + u'}}\n' + cid = cid + u'== {{int:license}} ==\n' + cid = cid + licensetemplate + u'\n' + cid = cid + u'\n' + cid = cid + self.getOriginalUploadLog(imagepage) + cid = cid + u'__NOTOC__\n' + if categories.strip()==u'': + cid = cid + u'{{Subst:Unc}}' + else: + cid = cid + categories + return cid + + def getOriginalUploadLog(self, imagepage): + ''' + Get the original upload log to put at the bottom of the image description page at Commons. + ''' + filehistory = imagepage.getFileVersionHistory() + filehistory.reverse() + + site = imagepage.site() + lang = site.language() + family = site.family.name + + sourceimage = imagepage.site().get_address(imagepage.title()).replace(u'&redirect=no&useskin=monobook', u'') + + result = u'== {{Original upload log}} ==\n' + result = result + u'The original description page is/was [http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following user names refer to %(lang)s.%(family)s.\n' % {u'lang' : lang, u'family' : family , u'sourceimage' : sourceimage} + for (timestamp, username, resolution, size, comment) in filehistory: + date = datetime.strptime(timestamp, u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M') + result = result + u'* %(date)s [[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s (%(size)s bytes) \'\'<nowiki>%(comment)s</nowiki>\'\'\n' % { + u'lang' : lang, + u'family' : family , + u'date' : date, + u'username' : username, + u'resolution': resolution, + u'size': size, + u'comment' : comment} + + return result + + def tagNowcommons(self, imagepage, filename): + ''' + Tagged the imag which has been moved to Commons for deletion. + ''' if wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename).exists(): #Get a fresh copy, force to get the page so we dont run into edit conflicts imtxt=imagepage.get(force=True) @@ -554,27 +654,32 @@ wikipedia.showDiff(imagepage.get(), imtxt + addTemplate) imagepage.put(imtxt + addTemplate, comment = commentText) + def replaceUsage(self, imagepage, filename): + ''' + If the image is uploaded under a different name, replace all usage. + ''' + if imagepage.titleWithoutNamespace() != filename: gen = pagegenerators.FileLinksGenerator(imagepage) preloadingGen = pagegenerators.PreloadingGenerator(gen) - - #If the image is uploaded under a different name, replace all instances - if imagepage.titleWithoutNamespace() != filename: - if imagepage.site().language() in imageMoveMessage: - moveSummary = imageMoveMessage[imagepage.site().language()] % (imagepage.titleWithoutNamespace(), filename) - else: - moveSummary = imageMoveMessage['_default'] % (imagepage.titleWithoutNamespace(), filename) - imagebot = ImageRobot(generator = preloadingGen, oldImage = imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary, always = True, loose = True) - imagebot.run() + + if imagepage.site().language() in imageMoveMessage: + moveSummary = imageMoveMessage[imagepage.site().language()] % (imagepage.titleWithoutNamespace(), filename) + else: + moveSummary = imageMoveMessage['_default'] % (imagepage.titleWithoutNamespace(), filename) + imagebot = ImageRobot(generator = preloadingGen, oldImage = imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary, always = True, loose = True) + imagebot.run() +def main(args): + wikipedia.output(u'WARNING: This is an experimental bot') + wikipedia.output(u'WARNING: It will only work on self published work images') + wikipedia.output(u'WARNING: This bot is still full of bugs') + wikipedia.output(u'WARNING: Use at your own risk!') -def main(args): generator = None; - #newname = ""; - imagepage = None; always = False checkTemplate = True - imagerecat.initLists() + # Load a lot of default generators genFactory = pagegenerators.GeneratorFactory() @@ -590,16 +695,25 @@ pregenerator = pagegenerators.PreloadingGenerator(generator) - for page in pregenerator: - processImage(page, checkTemplate) + prefetchQueue = Queue(maxsize=50) + uploadQueue = Queue(maxsize=200) + imageFetcherThread = imageFetcher(pregenerator, prefetchQueue) + userInteractionThread = userInteraction(prefetchQueue, uploadQueue) + uploaderThread = uploader(uploadQueue) - wikipedia.output(u'Still ' + str(threading.activeCount()) + u' active threads, lets wait') - for openthread in threading.enumerate(): - if openthread != threading.currentThread(): - openthread.join() - wikipedia.output(u'All threads are done') + imageFetcherThread.daemon=False + userInteractionThread.daemon=False + uploaderThread.daemon=False + + if not checkTemplate: + uploaderThread.nochecktemplate() + + fetchDone = imageFetcherThread.start() + userDone = userInteractionThread.start() + uploadDone = uploaderThread.start() + if __name__ == "__main__": try: main(sys.argv[1:])

1 0

SVN: [8387] trunk/pywikipedia/cosmetic_changes.py
by xqt＠svn.wikimedia.org 07 Aug '10

07 Aug '10

Revision: 8387 Author: xqt Date: 2010-08-07 13:37:23 +0000 (Sat, 07 Aug 2010) Log Message: ----------- regex fix for nn-comment Modified Paths: -------------- trunk/pywikipedia/cosmetic_changes.py Modified: trunk/pywikipedia/cosmetic_changes.py =================================================================== --- trunk/pywikipedia/cosmetic_changes.py 2010-08-07 13:35:28 UTC (rev 8386) +++ trunk/pywikipedia/cosmetic_changes.py 2010-08-07 13:37:23 UTC (rev 8387) @@ -372,7 +372,7 @@ # nn got a message between the categories and the iw's # and they want to keep it there, first remove it if self.site.language()=='nn': - regex = re.compile('()') + regex = re.compile('()') found = regex.findall(text) if found: if pywikibot.verbose:

1 0

SVN: [8386] trunk/pywikipedia/add_text.py
by xqt＠svn.wikimedia.org 07 Aug '10

07 Aug '10

Revision: 8386 Author: xqt Date: 2010-08-07 13:35:28 +0000 (Sat, 07 Aug 2010) Log Message: ----------- remove old nn-comment - bug #2819556 (code recycled from cosmetic_changes.py) Modified Paths: -------------- trunk/pywikipedia/add_text.py Modified: trunk/pywikipedia/add_text.py =================================================================== --- trunk/pywikipedia/add_text.py 2010-08-07 13:35:10 UTC (rev 8385) +++ trunk/pywikipedia/add_text.py 2010-08-07 13:35:28 UTC (rev 8386) @@ -243,18 +243,24 @@ interwikiInside = pywikibot.getLanguageLinks(newtext, site) # Removing the interwiki newtext = pywikibot.removeLanguageLinks(newtext, site) - # nn got a message between the categories and the iw's and they want to - # keep it there, first remove it + # nn got a message between the categories and the iw's + # and they want to keep it there, first remove it + hasCommentLine = False if (site.language()==u'nn'): - newtext = newtext.replace(nn_iw_msg, '') + regex = re.compile('()') + found = regex.findall(newtext) + if found: + hasCommentLine = True + newtext = regex.sub('', newtext) + # Adding the text newtext += u"\n%s" % addText # Reputting the categories newtext = pywikibot.replaceCategoryLinks(newtext, categoriesInside, site, True) #Put the nn iw message back - if (site.language()==u'nn'): - newtext = newtext + u'\n' + nn_iw_msg + if site.language()==u'nn' and (interwikiInside or hasCommentLine): + newtext = newtext + u'\r\n\r\n' + nn_iw_msg # Dealing the stars' issue allstars = [] starstext = pywikibot.removeDisabledParts(text)

1 0

SVN: [8385] trunk/pywikipedia/imagecopy_enwp.py
by multichill＠svn.wikimedia.org 07 Aug '10

07 Aug '10

Revision: 8385 Author: multichill Date: 2010-08-07 13:35:10 +0000 (Sat, 07 Aug 2010) Log Message: ----------- A couple of improvements Modified Paths: -------------- trunk/pywikipedia/imagecopy_enwp.py Modified: trunk/pywikipedia/imagecopy_enwp.py =================================================================== --- trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 12:47:21 UTC (rev 8384) +++ trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 13:35:10 UTC (rev 8385) @@ -120,12 +120,13 @@ (u'\{\{Multilicense replacing placeholder new(\|class=[^\}]+)?\}\}', u'{{Self|GFDL|Cc-by-sa-3.0,2.5,2.0,1.0|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), ] -sourceGarbage = [u'== Summary ==', - u'== Licensing:? ==', +sourceGarbage = [u'==\s*Summary\s*==', + u'==\s*Licensing:?\s*==', + u'\{\{(Copy to Wikimedia Commons|Move to Commons|Move to commons|Move to Wikimedia Commons|Copy to commons|Mtc|MtC|MTC|CWC|CtWC|CTWC|Ctwc|Tocommons|Copy to Commons|To Commons|Movetocommons|Move to Wikimedia commons|Move-to-commons|Commons ok|ToCommons|To commons|MoveToCommons|Copy to wikimedia commons|Upload to commons|CopyToCommons|Copytocommons|MITC|MovetoCommons|Do move to Commons)\}\}' ] class Tkdialog: - def __init__(self, imagepage, currentcontent): + def __init__(self, imagepage, description, date, source, author, licensetemplate, categories): self.root=Tk() #"%dx%d%+d%+d" % (width, height, xoffset, yoffset) #Always appear the same size and in the bottom-left corner @@ -144,12 +145,12 @@ self.filename = imagepage.titleWithoutNamespace() - (self.description, - self.date, - self.source, - self.author, - self.licensetemplate, - self.categories) = currentcontent + self.description = description + self.date = date + self.source = source + self.author = author + self.licensetemplate = licensetemplate + self.categories = categories self.skip = False self.old_description_label=Label(self.root,text=u'The old description was : ') @@ -282,7 +283,7 @@ Build a new description based on the imagepage ''' if u'{{Information' in imagepage.get() or u'{{information' in imagepage.get(): - (description, date, source, author) = getNewFieldsFromInformation(imagepage) + (description, date, source, author) = getNewFieldsFromInformation(imagepage) else: (description, date, source, author) = getNewFieldsFromFreetext(imagepage) @@ -301,6 +302,7 @@ other_versions = u'' text = imagepage.get() # Need to add the permission field + # Need to use pywikipedia template parser code regexes =[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}', ] @@ -310,9 +312,17 @@ match =re.search(regex, text, re.IGNORECASE|re.DOTALL) if match: description = convertLinks(match.group(u'description').strip(), imagepage.site()) + date = match.group(u'date').strip() + if date == u'': + date = getUploadDate(imagepage) + source = getSource(imagepage, source=convertLinks(match.group(u'source').strip(), imagepage.site())) + author = convertLinks(match.group(u'author').strip(), imagepage.site()) + if author == u'': + author = getAuthorText(imagepage) + if u'permission' in match.groupdict(): permission = convertLinks(match.group(u'permission').strip(), imagepage.site()) if u'other_versions' in match.groupdict(): @@ -485,67 +495,76 @@ #First do autoskip. if doiskip(imagepage): - wikipedia.output("Skipping " + page.title()) - skip = True - else: - currentcontent = getNewFields(imagepage) + wikipedia.output(u'Skipping %s : Got a template on the skip list.' % page.title()) + return False + + text = imagepage.get() + foundMatch = False + for (regex, replacement) in licenseTemplates: + match = re.search(regex, text, re.IGNORECASE) + if match: + foundMatch = True + if not foundMatch: + wikipedia.output(u'Skipping %s : No suitable license template was found.' % page.title()) + return False + + (description, date, source, author, licensetemplate, categories) = getNewFields(imagepage) - while True: - # Do the Tkdialog to accept/reject and change te name - (filename, description, date, source, author, licensetemplate, categories, skip)=Tkdialog(imagepage, currentcontent).getnewmetadata() + while True: + # Do the Tkdialog to accept/reject and change te name + (filename, description, date, source, author, licensetemplate, categories, skip)=Tkdialog(imagepage, description, date, source, author, licensetemplate, categories).getnewmetadata() - if skip: - wikipedia.output('Skipping this image') - break - - # Check if the image already exists - CommonsPage=wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename) - if not CommonsPage.exists(): - break - else: - wikipedia.output('Image already exists, pick another name or skip this image') - # We dont overwrite images, pick another name, go to the start of the loop - - if not skip: - cid = buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories, checkTemplate) - wikipedia.output(cid) - bot = UploadRobot(url=imagepage.fileUrl(), description=cid, useFilename=filename, keepFilename=True, verifyDescription=False, ignoreWarning = True, targetSite = wikipedia.getSite('commons', 'commons')) - bot.run() - - if wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename).exists(): - #Get a fresh copy, force to get the page so we dont run into edit conflicts - imtxt=imagepage.get(force=True) + if skip: + wikipedia.output(u'Skipping %s : User pressed skip.' % page.title()) + return False + + # Check if the image already exists + CommonsPage=wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename) + if not CommonsPage.exists(): + break + else: + wikipedia.output('Image already exists, pick another name or skip this image') + # We dont overwrite images, pick another name, go to the start of the loop + + cid = buildNewImageDescription(imagepage, description, date, source, author, licensetemplate, categories, checkTemplate) + wikipedia.output(cid) + bot = UploadRobot(url=imagepage.fileUrl(), description=cid, useFilename=filename, keepFilename=True, verifyDescription=False, ignoreWarning = True, targetSite = wikipedia.getSite('commons', 'commons')) + bot.run() + + if wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' + filename).exists(): + #Get a fresh copy, force to get the page so we dont run into edit conflicts + imtxt=imagepage.get(force=True) - #Remove the move to commons templates - if imagepage.site().language() in moveToCommonsTemplate: - for moveTemplate in moveToCommonsTemplate[imagepage.site().language()]: - imtxt = re.sub(u'(?i)\{\{' + moveTemplate + u'[^\}]*\}\}', u'', imtxt) + #Remove the move to commons templates + if imagepage.site().language() in moveToCommonsTemplate: + for moveTemplate in moveToCommonsTemplate[imagepage.site().language()]: + imtxt = re.sub(u'(?i)\{\{' + moveTemplate + u'[^\}]*\}\}', u'', imtxt) - #add {{NowCommons}} - if imagepage.site().language() in nowCommonsTemplate: - addTemplate = nowCommonsTemplate[imagepage.site().language()] % filename - else: - addTemplate = nowCommonsTemplate['_default'] % filename + #add {{NowCommons}} + if imagepage.site().language() in nowCommonsTemplate: + addTemplate = nowCommonsTemplate[imagepage.site().language()] % filename + else: + addTemplate = nowCommonsTemplate['_default'] % filename - if imagepage.site().language() in nowCommonsMessage: - commentText = nowCommonsMessage[imagepage.site().language()] - else: - commentText = nowCommonsMessage['_default'] + if imagepage.site().language() in nowCommonsMessage: + commentText = nowCommonsMessage[imagepage.site().language()] + else: + commentText = nowCommonsMessage['_default'] - wikipedia.showDiff(imagepage.get(), imtxt + addTemplate) - imagepage.put(imtxt + addTemplate, comment = commentText) + wikipedia.showDiff(imagepage.get(), imtxt + addTemplate) + imagepage.put(imtxt + addTemplate, comment = commentText) - gen = pagegenerators.FileLinksGenerator(imagepage) - preloadingGen = pagegenerators.PreloadingGenerator(gen) + gen = pagegenerators.FileLinksGenerator(imagepage) + preloadingGen = pagegenerators.PreloadingGenerator(gen) - #If the image is uploaded under a different name, replace all instances - if imagepage.titleWithoutNamespace() != filename: - if imagepage.site().language() in imageMoveMessage: - moveSummary = imageMoveMessage[imagepage.site().language()] % (imagepage.titleWithoutNamespace(), filename) - else: - moveSummary = imageMoveMessage['_default'] % (imagepage.titleWithoutNamespace(), filename) - imagebot = ImageRobot(generator = preloadingGen, oldImage = imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary, always = True, loose = True) - imagebot.run() + #If the image is uploaded under a different name, replace all instances + if imagepage.titleWithoutNamespace() != filename: + if imagepage.site().language() in imageMoveMessage: + moveSummary = imageMoveMessage[imagepage.site().language()] % (imagepage.titleWithoutNamespace(), filename) + else: + moveSummary = imageMoveMessage['_default'] % (imagepage.titleWithoutNamespace(), filename) + imagebot = ImageRobot(generator = preloadingGen, oldImage = imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary, always = True, loose = True) + imagebot.run()

1 0

SVN: [8384] trunk/pywikipedia/featured.py
by xqt＠svn.wikimedia.org 07 Aug '10

07 Aug '10

Revision: 8384 Author: xqt Date: 2010-08-07 12:47:21 +0000 (Sat, 07 Aug 2010) Log Message: ----------- Japanese localisation - patch request bug #3037570 Modified Paths: -------------- trunk/pywikipedia/featured.py Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2010-08-07 11:14:30 UTC (rev 8383) +++ trunk/pywikipedia/featured.py 2010-08-07 12:47:21 UTC (rev 8384) @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python # -*- coding: utf-8 -*- """ This script understands various command-line arguments: @@ -85,7 +85,7 @@ 'hu': u'Bot: a(z) [[%s:%s]] kiemelt szócikk', 'ia': u'Robot: Ligamine verso articulo eminente [[%s:%s]]', 'it': u'Bot: collegamento articolo in vetrina [[%s:%s]]', - 'ja': u'ロボットによる: 秀逸な項目へのリンク [[%s:%s]]', + 'ja': u'ロボットによる: 秀逸な記事へのリンク [[%s:%s]]', 'ka': u'ბოტი: რჩეული სტატიის ბმული გვერდისათვის [[%s:%s]]', 'ko': u'로봇: 알찬 글 [[%s:%s]] 를 가리키는 링크',#로봇이：? 'ksh':u'bot: [[%s:%s]] ess_enen ußjezëijshneten Atikkel', @@ -119,6 +119,7 @@ 'fa': u'ربات: [[%s:%s]] یک مقاله خوب است', 'fi': u'Botti: [[%s:%s]] on hyvä artikkeli', 'fr': u'Bot: Lien BA pour [[%s:%s]]', + 'ja': u'ロボットによる: 良質な記事へのリンク [[%s:%s]]', 'ksh':u'bot: [[%s:%s]] ess_enen jooden Atikkel', 'no': u'bot: [[%s:%s]] er en anbefalt artikkel', 'nn': u'bot: [[%s:%s]] er ein god artikkel', @@ -136,6 +137,7 @@ 'es': u'Bot: Enlace a lista destacado para: [[%s:%s]]', 'fa': u'ربات: [[%s:%s]] یک فهرست برگزیده است', 'fi': u'Botti: [[%s:%s]] on suositeltu luettelo', + 'ja': u'ロボットによる: 秀逸な一覧へのリンク [[%s:%s]]', 'ksh':u'bot: [[%s:%s]] ess_en joode Leß', 'pt': u'Bot: [[%s:%s]] é uma lista destacada', 'sv': u'Bot: [[%s:%s]] är en utmärkt list', @@ -323,7 +325,7 @@ 'id': (BACK,u"Artikel bagus"), #'id': (CAT, u"Artikel bagus"), 'is': (CAT, u"Wikipedia:Gæðagreinar"), - 'ja': (CAT, u"おすすめ記事"), + 'ja': (BACK,u"Good article"), 'ksh':(CAT, u"Joode Aatikkel"), 'lt': (CAT, u"Vertingi straipsniai"), 'lv': (CAT, u"Labi raksti"),

1 0

← Newer
1
2
3
4
5
6
7
8
9
10
Older →

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn August 2010