Pywikipedia-svn

pywikipedia-svn@lists.wikimedia.org

5163 discussions

SVN: [11622] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 07 Jun '13

07 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11622 Revision: 11622 Author: drtrigon Date: 2013-06-07 18:21:31 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; format conversion in downloadImage and _detect_Properties_PIL re-aranged Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-07 14:53:04 UTC (rev 11621) +++ trunk/pywikipedia/catimages.py 2013-06-07 18:21:31 UTC (rev 11622) @@ -146,124 +146,36 @@ class UnknownFile(object): def __init__(self, filename, *args, **kwargs): - self.filename = filename + self.filename = filename + self.image_size = (None, None) + self._info = {} - def getProperties(self): - # check/look into the file by midnight commander (mc) - # https://pypi.python.org/pypi/hachoir-metadata - #self._detect_HeaderAndMetadata() + def __enter__(self): + return self - # get mime-type file-size, ... + def __exit__(self, type, value, traceback): pass + def getProperties(self): + result = {} + result.update( self._detect_HeaderAndMetadata() ) # Metadata + result.update( self._detect_Properties_PIL() ) # Properties + return result + def getFeatures(self): - pass + pywikibot.warning(u"File format '%s/%s' not supported (yet)!" % tuple(self.image_mime[:2])) + def _detect_HeaderAndMetadata(self): + # check/look into the file by midnight commander (mc) + # https://pypi.python.org/pypi/hachoir-metadata + return {} + def _detect_Properties_PIL(self): - """Retrieve as much file property info possible, especially the same - as commons does in order to compare if those libraries (ImageMagick, - ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} - if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' - # similar to PDF page count OR use BeautifulSoup - svgcountpages = re.compile("<page>") - pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + # get mime-type file-size, ... + return {} - #svg = rsvg.Handle(self.image_path) - # http://validator.w3.org/docs/api.html#libs - # http://pypi.python.org/pypi/py_w3c/ - vld = HTMLValidator() - valid = u'SVG' - try: - vld.validate(self.image.fileUrl()) - valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') - except urllib2.URLError: - pass - except ValidationFault: - pass - #print vld.errors, vld.warnings - - #self.image_size = (svg.props.width, svg.props.height) - - result = { 'Format': valid, - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - # may be set {{validSVG}} also or do something in bot template to - # recognize 'Format=SVG (valid)' ... - elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' - # http://code.activestate.com/recipes/496837-count-pdf-pages/ - #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) - rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 - pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) - - result = { 'Format': u'PDF', - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'x-xcf': # MIME: 'image/x-xcf; charset=binary' - result = { 'Format': u'%s' % self.image_mime[1].upper() } - # DO NOT use ImageMagick (identify) instead of PIL to get these info !! - elif (self.image_mime[0] == 'image') and \ - (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... - try: - i = Image.open(self.image_path) - except IOError: - pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') - return {'Properties': [result]} - - # http://mail.python.org/pipermail/image-sig/1999-May/000740.html - pc=0 # count number of pages - while True: - try: - i.seek(pc) - except EOFError: - break - pc+=1 - i.seek(0) # restore default - - # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles - # python-lcms (littlecms) may be freeimage library - #icc = i.app['APP2'] # jpeg - #icc = i.tag[34675] # tiff - #icc = re.sub('[^%s]'%string.printable, ' ', icc) - ## more image formats and more post-processing needed... - - #self.image_size = i.size - - result = { #'bands': i.getbands(), - #'bbox': i.getbbox(), - 'Format': i.format, - 'Mode': i.mode, - #'info': i.info, - #'stat': os.stat(self.image_path), - 'Palette': str(len(i.palette.palette)) if i.palette else u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' - # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - #print d - result['Format'] = u'%s' % d['format']['format_name'].upper() - elif self.image_mime[1] == 'midi': # MIME: 'audio/midi; charset=binary' - result['Format'] = u'%s' % self.image_mime[1].upper() - # djvu: python-djvulibre or python-djvu for djvu support - # http://pypi.python.org/pypi/python-djvulibre/0.3.9 - else: - pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') - return {'Properties': [result]} - - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) - - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - - class JpegFile(UnknownFile): # for '_detect_Trained_CV' cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), @@ -280,12 +192,25 @@ # ('Hands' does not behave very well, in fact it detects any kind of skin and other things...) #(u'Aeroplanes', 'haarcascade_aeroplane.xml'),] # e.g. for 'Category:Unidentified aircraft' - def getProperties(self): - pass + def __init__(self, filename, *args, **kwargs): + UnknownFile.__init__(self, filename) - # Image size - self._detect_Properties_PIL() + self.image_filename = os.path.split(self.filename)[-1] + self.image_fileext = os.path.splitext(self.image_filename)[1] + self.image_path = os.path.join(scriptdir, ('cache/' + self.image_filename[-128:])) + self.image_path_JPEG = self.image_path + '.jpg' + self._convert() + + def __exit__(self, type, value, traceback): + if os.path.exists(self.image_path): + os.remove( self.image_path ) + if os.path.exists(self.image_path_JPEG): + os.remove( self.image_path_JPEG ) + #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") + #if os.path.exists(image_path_new): + # os.remove( image_path_new ) + def getFeatures(self): # Faces (extract EXIF data) self._detect_Faces_EXIF() @@ -336,6 +261,87 @@ # general file EXIF history information self._detect_History_EXIF() + # supports a lot of different file types thanks to PIL + def _convert(self): + try: + im = Image.open(self.image_path) # might be png, gif etc, for instance + #im.thumbnail(size, Image.ANTIALIAS) # size is 640x480 + im.convert('RGB').save(self.image_path_JPEG, "JPEG") + + self.image_size = im.size + except IOError, e: + if 'image file is truncated' in str(e): + # im object has changed due to exception raised + im.convert('RGB').save(self.image_path_JPEG, "JPEG") + + self.image_size = im.size + else: + try: + # since opencv might still work, try this as fall-back + img = cv2.imread( self.image_path, cv.CV_LOAD_IMAGE_COLOR ) + cv2.imwrite(self.image_path_JPEG, img) + + self.image_size = (img.shape[1], img.shape[0]) + except: + if os.path.exists(self.image_path_JPEG): + os.remove(self.image_path_JPEG) + self.image_path_JPEG = self.image_path + except: + self.image_path_JPEG = self.image_path + + # FULL TIFF support (e.g. group4) + # http://code.google.com/p/pylibtiff/ + + # MIME: 'image/jpeg; charset=binary', ... + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + try: + i = Image.open(self.image_path) + except IOError: + pywikibot.warning(u'unknown file type [JpegFile]') + return {'Properties': [result]} + + # http://mail.python.org/pipermail/image-sig/1999-May/000740.html + pc=0 # count number of pages + while True: + try: + i.seek(pc) + except EOFError: + break + pc+=1 + i.seek(0) # restore default + + # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles + # python-lcms (littlecms) may be freeimage library + #icc = i.app['APP2'] # jpeg + #icc = i.tag[34675] # tiff + #icc = re.sub('[^%s]'%string.printable, ' ', icc) + ## more image formats and more post-processing needed... + + #self.image_size = i.size + + result = { #'bands': i.getbands(), + #'bbox': i.getbbox(), + 'Format': i.format, + 'Mode': i.mode, + #'info': i.info, + #'stat': os.stat(self.image_path), + 'Palette': str(len(i.palette.palette)) if i.palette else u'-', + 'Pages': pc, } + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html def _detect_Faces_CV(self): @@ -2477,15 +2483,162 @@ class TiffFile(JpegFile): pass + class XcfFile(JpegFile): - pass + def _convert(self): + # Very few programs other than GIMP read XCF files. This is by design + # from the GIMP developers, the format is not really documented or + # supported as a general-purpose file format. + # Commons uses ImageMagick, thus we have EXACTLY THE SAME support! + # (can also be a drawback, e.g. when the library is buggy...) + proc = Popen("convert %s %s" % (self.image_path, self.image_path_JPEG), + shell=True, stderr=PIPE)#.stderr.read() + proc.wait() + if proc.returncode != 0: + raise ImportError("convert (ImageMagick) not found (may be other error occured)!") + elif proc.returncode: + self.image_path_JPEG = self.image_path + #data = Popen("identify -verbose info: %s" % self.image_path, + # shell=True, stderr=PIPE).stderr.read() + #print data + if not os.path.exists(self.image_path_JPEG): + # xcf can have more than 1 layer/page like gif, tiff, and movies... + self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') + self.image_size = Image.open(self.image_path_JPEG).size + + # MIME: 'image/x-xcf; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + result = { 'Format': u'%s' % self.image_mime[1].upper() } + # DO NOT use ImageMagick (identify) instead of PIL to get these info !! + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + class SvgFile(JpegFile): - pass + def _convert(self): + # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) + # (Mediawiki uses librsvg too: http://commons.wikimedia.org/wiki/SVG#SVGs_in_MediaWiki) + # http://stackoverflow.com/questions/6589358/convert-svg-to-png-in-python + # http://cairographics.org/pythoncairopil/ + # http://cairographics.org/pyrsvg/ + # http://stackoverflow.com/questions/9166400/convert-rgba-png-to-rgb-with-pil + try: + svg = rsvg.Handle(self.image_path) + img = cairo.ImageSurface(cairo.FORMAT_ARGB32, svg.props.width, svg.props.height) + ctx = cairo.Context(img) + svg.render_cairo(ctx) + #img.write_to_png("svg.png") + #Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), + # img.get_data(),"raw","RGBA",0,1).save(self.image_path_JPEG, "JPEG") + png = Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), + img.get_data(),"raw","RGBA",0,1) + background = Image.new("RGB", png.size, (255, 255, 255)) + background.paste(png, mask=png.split()[3]) # 3 is the alpha channel + background.save(self.image_path_JPEG, "JPEG") + self.image_size = (svg.props.width, svg.props.height) + except MemoryError: + self.image_path_JPEG = self.image_path + except SystemError: + self.image_path_JPEG = self.image_path + + # MIME: 'application/xml; charset=utf-8' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # similar to PDF page count OR use BeautifulSoup + svgcountpages = re.compile("<page>") + pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + + #svg = rsvg.Handle(self.image_path) + + # http://validator.w3.org/docs/api.html#libs + # http://pypi.python.org/pypi/py_w3c/ + vld = HTMLValidator() + valid = u'SVG' + try: + vld.validate(self.image.fileUrl()) + valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') + except urllib2.URLError: + pass + except ValidationFault: + pass + #print vld.errors, vld.warnings + + #self.image_size = (svg.props.width, svg.props.height) + + result = { 'Format': valid, + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + # may be set {{validSVG}} also or do something in bot template to + # recognize 'Format=SVG (valid)' ... + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + class PdfFile(JpegFile): - pass + def _convert(self): +# self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) + # PDF: support extract text and images + # (Mediawiki uses ghostscript: https://www.mediawiki.org/wiki/Extension:PdfHandler#Pre-requisites) + # http://vermeulen.ca/python-pdf.html + # http://code.activestate.com/recipes/511465-pure-python-pdf-to-text-converte… + # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… + if self.image_fileext == u'.pdf': + pass + + # MIME: 'application/pdf; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # http://code.activestate.com/recipes/496837-count-pdf-pages/ + #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) + rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 + pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) + + result = { 'Format': u'PDF', + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + #class DjvuFile(JpegFile): # pass @@ -2498,6 +2651,27 @@ # general audio feature extraction # self._detect_AudioFeatures_YAAFE() + # MIME: 'application/ogg; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + #print d + result['Format'] = u'%s' % d['format']['format_name'].upper() + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + def _detect_Streams_FFMPEG(self): self._info['Streams'] = [] @@ -2696,12 +2870,6 @@ class MidiFile(UnknownFile): - def getProperties(self): - result = {} - result.update( self._detect_HeaderAndMetadata() ) # Metadata - result.update( self._detect_Properties_PIL() ) # Properties - return result - def getFeatures(self): result = {} result.update( self._detect_AudioFeatures_MUSIC21() ) # Audio @@ -2739,6 +2907,24 @@ self._info['Metadata'] = [result] return {'Metadata': [result]} + # MIME: 'audio/midi; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + result['Format'] = u'%s' % self.image_mime[1].upper() + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + # midi audio feature extraction def _detect_AudioFeatures_MUSIC21(self): # skip file formats not supported @@ -2805,6 +2991,8 @@ ( 'image', 'x-xcf'): XcfFile, ( 'image', 'svg+xml'): SvgFile, ('application', 'pdf'): PdfFile, +# djvu: python-djvulibre or python-djvu for djvu support +# http://pypi.python.org/pypi/python-djvulibre/0.3.9 # ( 'image', 'vnd.djvu'): DjvuFile, ('application', 'ogg'): OggFile, ( 'audio', 'midi'): MidiFile,} @@ -3257,12 +3445,10 @@ #print self.image_path pywikibot.output(u'Processing media %s ...' % self.image.title(asLink=True)) - self.image_filename = os.path.split(self.image.fileUrl())[-1] - self.image_fileext = os.path.splitext(self.image_filename)[1] - self.image_path = urllib2.quote(os.path.join(scriptdir, ('cache/' + self.image_filename[-128:]))) - - self.image_path_JPEG = self.image_path + u'.jpg' - + image_filename = os.path.split(self.image.fileUrl())[-1] + image_fileext = os.path.splitext(image_filename)[1] + self.image_path = urllib2.quote(os.path.join(scriptdir, ('cache/' + image_filename[-128:]))) + self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) #print self._wikidata #print self._wikidata['mime'] @@ -3270,7 +3456,7 @@ #print self._wikidata['metadata'] #for item in self._wikidata['metadata']: # print item['name'], item['value'] - + if not os.path.exists(self.image_path): pywikibot.get_throttle() f_url, data = self.site.getUrl(self.image.fileUrl(), no_hostname=True, @@ -3279,7 +3465,7 @@ # (allows to re-read from back_response) data = f_url.read() del f_url # free some memory (no need to keep a copy...) - + f = open(self.image_path, 'wb') f.write( data ) f.close() @@ -3290,95 +3476,9 @@ self.image_mime = re.split('[/;\s]', m.file(self.image_path)) #self.image_size = (None, None) mime = mimetypes.guess_all_extensions('%s/%s' % tuple(self.image_mime[0:2])) - if mime and (self.image_fileext.lower() not in mime): + if mime and (image_fileext.lower() not in mime): pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) - # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) - # (Mediawiki uses librsvg too: http://commons.wikimedia.org/wiki/SVG#SVGs_in_MediaWiki) - # http://stackoverflow.com/questions/6589358/convert-svg-to-png-in-python - # http://cairographics.org/pythoncairopil/ - # http://cairographics.org/pyrsvg/ - # http://stackoverflow.com/questions/9166400/convert-rgba-png-to-rgb-with-pil - self.image_size = (None, None) - if self.image_fileext == u'.svg': - try: - svg = rsvg.Handle(self.image_path) - img = cairo.ImageSurface(cairo.FORMAT_ARGB32, svg.props.width, svg.props.height) - ctx = cairo.Context(img) - svg.render_cairo(ctx) - #img.write_to_png("svg.png") - #Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), - # img.get_data(),"raw","RGBA",0,1).save(self.image_path_JPEG, "JPEG") - png = Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), - img.get_data(),"raw","RGBA",0,1) - background = Image.new("RGB", png.size, (255, 255, 255)) - background.paste(png, mask=png.split()[3]) # 3 is the alpha channel - background.save(self.image_path_JPEG, "JPEG") - - self.image_size = (svg.props.width, svg.props.height) - except MemoryError: - self.image_path_JPEG = self.image_path - except SystemError: - self.image_path_JPEG = self.image_path - elif self.image_fileext == u'.xcf': - # Very few programs other than GIMP read XCF files. This is by design - # from the GIMP developers, the format is not really documented or - # supported as a general-purpose file format. - # Commons uses ImageMagick, thus we have EXACTLY THE SAME support! - # (can also be a drawback, e.g. when the library is buggy...) - proc = Popen("convert %s %s" % (self.image_path, self.image_path_JPEG), - shell=True, stderr=PIPE)#.stderr.read() - proc.wait() - if proc.returncode == 127: - raise ImportError("convert (ImageMagick) not found!") - elif proc.returncode: - self.image_path_JPEG = self.image_path - - #data = Popen("identify -verbose info: %s" % self.image_path, - # shell=True, stderr=PIPE).stderr.read() - #print data - if not os.path.exists(self.image_path_JPEG): - # xcf can have more than 1 layer/page like gif, tiff, and movies... - self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') - self.image_size = Image.open(self.image_path_JPEG).size - else: - try: - im = Image.open(self.image_path) # might be png, gif etc, for instance - #im.thumbnail(size, Image.ANTIALIAS) # size is 640x480 - im.convert('RGB').save(self.image_path_JPEG, "JPEG") - - self.image_size = im.size - except IOError, e: - if 'image file is truncated' in str(e): - # im object has changed due to exception raised - im.convert('RGB').save(self.image_path_JPEG, "JPEG") - - self.image_size = im.size - else: - try: - # since opencv might still work, try this as fall-back - img = cv2.imread( self.image_path, cv.CV_LOAD_IMAGE_COLOR ) - cv2.imwrite(self.image_path_JPEG, img) - - self.image_size = (img.shape[1], img.shape[0]) - except: - if os.path.exists(self.image_path_JPEG): - os.remove(self.image_path_JPEG) - self.image_path_JPEG = self.image_path - except: - self.image_path_JPEG = self.image_path - - # PDF: support extract text and images - # (Mediawiki uses ghostscript: https://www.mediawiki.org/wiki/Extension:PdfHandler#Pre-requisites) - # http://vermeulen.ca/python-pdf.html - # http://code.activestate.com/recipes/511465-pure-python-pdf-to-text-converte… - # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… - if self.image_fileext == u'.pdf': - pass - - # FULL TIFF support (e.g. group4) - # http://code.google.com/p/pylibtiff/ - # LOOK ALSO AT: checkimages.CatImagesBot.checkStep # (and category scripts/bots too...) def checkStep(self): @@ -3564,13 +3664,14 @@ return u"\n".join( ret ) def clean_cache(self): - if os.path.exists(self.image_path): - os.remove( self.image_path ) - if os.path.exists(self.image_path_JPEG): - os.remove( self.image_path_JPEG ) - #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") - #if os.path.exists(image_path_new): - # os.remove( image_path_new ) +# if os.path.exists(self.image_path): +# os.remove( self.image_path ) +# if os.path.exists(self.image_path_JPEG): +# os.remove( self.image_path_JPEG ) +# #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") +# #if os.path.exists(image_path_new): +# # os.remove( image_path_new ) + pass # LOOK ALSO AT: checkimages.CatImagesBot.report def report(self): @@ -3718,17 +3819,18 @@ # split detection and extraction according to file types; JpegFile, ... TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) - tf = TypeFile(self.image_path) - import copy - tf.__dict__.update(copy.deepcopy(self.__dict__)) - for func in ['getProperties', 'getFeatures']: - result = getattr(tf, func)() - if result: - self._info.update(result) -# print self._info -# print tf._info - #print tf.__dict__ - self._info = tf._info + with TypeFile(self.image_path, self.image_mime) as tf: + import copy + tf.__dict__.update(copy.deepcopy(self.__dict__)) + for func in ['getProperties', 'getFeatures']: + result = getattr(tf, func)() + if result: + self._info.update(result) + print self._info + print tf._info + #print tf.__dict__ + self._info = tf._info + self.image_size = tf.image_size def _existInformation(self, info, ignore = ['Properties', 'ColorAverage']): result = []

1 0

SVN: [11621] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 07 Jun '13

07 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11621 Revision: 11621 Author: drtrigon Date: 2013-06-07 14:53:04 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; start of code structure clean-up Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-06 21:19:06 UTC (rev 11620) +++ trunk/pywikipedia/catimages.py 2013-06-07 14:53:04 UTC (rev 11621) @@ -143,7 +143,199 @@ # all detection and recognition methods - bindings to other classes, modules and libs -class FileData(object): + +class UnknownFile(object): + def __init__(self, filename, *args, **kwargs): + self.filename = filename + self._info = {} + + def getProperties(self): + # check/look into the file by midnight commander (mc) + # https://pypi.python.org/pypi/hachoir-metadata + #self._detect_HeaderAndMetadata() + + # get mime-type file-size, ... + pass + + def getFeatures(self): + pass + + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' + # similar to PDF page count OR use BeautifulSoup + svgcountpages = re.compile("<page>") + pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + + #svg = rsvg.Handle(self.image_path) + + # http://validator.w3.org/docs/api.html#libs + # http://pypi.python.org/pypi/py_w3c/ + vld = HTMLValidator() + valid = u'SVG' + try: + vld.validate(self.image.fileUrl()) + valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') + except urllib2.URLError: + pass + except ValidationFault: + pass + #print vld.errors, vld.warnings + + #self.image_size = (svg.props.width, svg.props.height) + + result = { 'Format': valid, + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + # may be set {{validSVG}} also or do something in bot template to + # recognize 'Format=SVG (valid)' ... + elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' + # http://code.activestate.com/recipes/496837-count-pdf-pages/ + #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) + rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 + pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) + + result = { 'Format': u'PDF', + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + elif self.image_mime[1] == 'x-xcf': # MIME: 'image/x-xcf; charset=binary' + result = { 'Format': u'%s' % self.image_mime[1].upper() } + # DO NOT use ImageMagick (identify) instead of PIL to get these info !! + elif (self.image_mime[0] == 'image') and \ + (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... + try: + i = Image.open(self.image_path) + except IOError: + pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') + return {'Properties': [result]} + + # http://mail.python.org/pipermail/image-sig/1999-May/000740.html + pc=0 # count number of pages + while True: + try: + i.seek(pc) + except EOFError: + break + pc+=1 + i.seek(0) # restore default + + # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles + # python-lcms (littlecms) may be freeimage library + #icc = i.app['APP2'] # jpeg + #icc = i.tag[34675] # tiff + #icc = re.sub('[^%s]'%string.printable, ' ', icc) + ## more image formats and more post-processing needed... + + #self.image_size = i.size + + result = { #'bands': i.getbands(), + #'bbox': i.getbbox(), + 'Format': i.format, + 'Mode': i.mode, + #'info': i.info, + #'stat': os.stat(self.image_path), + 'Palette': str(len(i.palette.palette)) if i.palette else u'-', + 'Pages': pc, } + elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' + # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + #print d + result['Format'] = u'%s' % d['format']['format_name'].upper() + elif self.image_mime[1] == 'midi': # MIME: 'audio/midi; charset=binary' + result['Format'] = u'%s' % self.image_mime[1].upper() + # djvu: python-djvulibre or python-djvu for djvu support + # http://pypi.python.org/pypi/python-djvulibre/0.3.9 + else: + pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') + return {'Properties': [result]} + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + +class JpegFile(UnknownFile): + # for '_detect_Trained_CV' + cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), + (u'Torsos', 'haarcascade_upperbody.xml'), + (u'Ears', 'haarcascade_mcs_leftear.xml'), + (u'Ears', 'haarcascade_mcs_rightear.xml'), + (u'Eyes', 'haarcascade_lefteye_2splits.xml'), # (http://yushiqi.cn/research/eyedetection) + (u'Eyes', 'haarcascade_righteye_2splits.xml'), # (http://yushiqi.cn/research/eyedetection) + #externals/opencv/haarcascades/haarcascade_mcs_lefteye.xml + #externals/opencv/haarcascades/haarcascade_mcs_righteye.xml + # (others include indifferent (left and/or right) and pair) + (u'Automobiles', 'cars3.xml'), # http://www.youtube.com/watch?v=c4LobbqeKZc + (u'Hands', '1256617233-2-haarcascade-hand.xml', 300.),] # http://www.andol.info/ + # ('Hands' does not behave very well, in fact it detects any kind of skin and other things...) + #(u'Aeroplanes', 'haarcascade_aeroplane.xml'),] # e.g. for 'Category:Unidentified aircraft' + + def getProperties(self): + pass + + # Image size + self._detect_Properties_PIL() + + def getFeatures(self): + # Faces (extract EXIF data) + self._detect_Faces_EXIF() + # Faces and eyes (opencv pre-trained haar) + self._detect_Faces_CV() +# TODO: test and use or switch off + # Face via Landmark(s) +# self._detect_FaceLandmark_xBOB() + # exclude duplicates (CV and EXIF) + faces = [item['Position'] for item in self._info['Faces']] + for i in self._util_merge_Regions(faces)[1]: + del self._info['Faces'][i] + + # Segments and colors + self._detect_SegmentColors_JSEGnPIL() + # Average color + self._detect_AverageColor_PILnCV() + + # People/Pedestrian (opencv pre-trained hog and haarcascade) + self._detect_People_CV() + + # Geometric object (opencv hough line, circle, edges, corner, ...) + self._detect_Geometry_CV() + + # general (opencv pre-trained, third-party and self-trained haar + # and cascade) classification + # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… + for cf in self.cascade_files: + self._detect_Trained_CV(*cf) + + # optical and other text recognition (tesseract & ocropus, ...) + self._detect_EmbeddedText_poppler() +# self._recognize_OpticalText_ocropus() + # (may be just classify as 'contains text', may be store text, e.g. to wikisource) + + # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) + self._recognize_OpticalCodes_dmtxNzbar() + + # Chessboard (opencv reference detector) + self._detect_Chessboard_CV() + + # general (self-trained) detection WITH classification + # BoW: uses feature detection (SIFT, SURF, ...) AND classification (SVM, ...) +# self._detectclassify_ObjectAll_CV() + # Wavelet: uses wavelet transformation AND classification (machine learning) +# self._detectclassify_ObjectAll_PYWT() + + # general file EXIF history information + self._detect_History_EXIF() + # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html def _detect_Faces_CV(self): @@ -157,7 +349,8 @@ # http://www.cognotics.com/opencv/servo_2007_series/part_4/index.html # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # https://code.ros.org/trac/opencv/browser/trunk/opencv_extra/testdata/gpu/ha… @@ -699,7 +892,8 @@ self._info['Geometry'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return result = self._util_get_Geometry_CVnSCIPY() @@ -970,7 +1164,8 @@ self._info['ColorRegions'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return try: @@ -1032,7 +1227,8 @@ self._info['ColorAverage'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return try: @@ -1050,109 +1246,6 @@ self._info['ColorAverage'] = [result] return - def _detect_Properties_PIL(self): - """Retrieve as much file property info possible, especially the same - as commons does in order to compare if those libraries (ImageMagick, - ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - self._info['Properties'] = [{'Format': u'-', 'Pages': 0}] - if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' - # similar to PDF page count OR use BeautifulSoup - svgcountpages = re.compile("<page>") - pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) - - #svg = rsvg.Handle(self.image_path) - - # http://validator.w3.org/docs/api.html#libs - # http://pypi.python.org/pypi/py_w3c/ - vld = HTMLValidator() - valid = u'SVG' - try: - vld.validate(self.image.fileUrl()) - valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') - except urllib2.URLError: - pass - except ValidationFault: - pass - #print vld.errors, vld.warnings - - #self.image_size = (svg.props.width, svg.props.height) - - result = { 'Format': valid, - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - # may be set {{validSVG}} also or do something in bot template to - # recognize 'Format=SVG (valid)' ... - elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' - # http://code.activestate.com/recipes/496837-count-pdf-pages/ - #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) - rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 - pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) - - result = { 'Format': u'PDF', - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - elif (self.image_mime[0] == 'image') and \ - (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... - try: - i = Image.open(self.image_path) - except IOError: - pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') - return - - # http://mail.python.org/pipermail/image-sig/1999-May/000740.html - pc=0 # count number of pages - while True: - try: - i.seek(pc) - except EOFError: - break - pc+=1 - i.seek(0) # restore default - - # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles - # python-lcms (littlecms) may be freeimage library - #icc = i.app['APP2'] # jpeg - #icc = i.tag[34675] # tiff - #icc = re.sub('[^%s]'%string.printable, ' ', icc) - ## more image formats and more post-processing needed... - - #self.image_size = i.size - - result = { #'bands': i.getbands(), - #'bbox': i.getbbox(), - 'Format': i.format, - 'Mode': i.mode, - #'info': i.info, - #'stat': os.stat(self.image_path), - 'Palette': str(len(i.palette.palette)) if i.palette else u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' - # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - #print d - result = { 'Format': u'%s' % d['format']['format_name'].upper() } - elif self.image_mime[0] == 'audio': # MIME: 'audio/midi; charset=binary' - result = {} - # djvu: python-djvulibre or python-djvu for djvu support - # http://pypi.python.org/pypi/python-djvulibre/0.3.9 - #elif self.image_fileext == u'.xcf' - # result = {} - # # DO NOT use ImageMagick (identify) instead of PIL to get these info !! - else: - pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') - return - - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) - - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return - # http://stackoverflow.com/questions/2270874/image-color-detection-using-pyth… # https://gist.github.com/1246268 # colormath-1.0.8/examples/delta_e.py, colormath-1.0.8/examples/conversions.py @@ -1413,7 +1506,8 @@ self._info[info_desc] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # http://tutorial-haartraining.googlecode.com/svn/trunk/data/haarcascades/ @@ -1634,7 +1728,8 @@ self._info['OpticalCodes'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # DataMatrix @@ -1734,7 +1829,8 @@ self._info['Chessboard'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return scale = 1. @@ -2306,45 +2402,6 @@ return self._buffer_FFMPEG - def _detect_Streams_FFMPEG(self): - self._info['Streams'] = [] - - # skip file formats that interfere and can cause strange results (pdf is oga?!) - # or file formats not supported (yet?) - if (self.image_mime[1] in ['pdf']) or (self.image_fileext in [u'.svg']): - return - - # audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - if not d: - return - - result = [] - for s in d['streams']: - #print s - if (s["codec_type"] == "video"): - rate = s["avg_frame_rate"] - dim = (int(s["width"]), int(s["height"])) - #asp = s["display_aspect_ratio"] - elif (s["codec_type"] == "audio"): -# switch this part off since 'ffprobe' (on toolserver) is too old -# rate = u'%s/%s/%s' % (s["channels"], s["sample_fmt"], s["sample_rate"]) - rate = u'%s/%s/%s' % (s["channels"], u'-', int(float(s["sample_rate"]))) - dim = None - elif (s["codec_type"] == "data"): - rate = None - dim = None - - result.append({ 'ID': int(s["index"]) + 1, - 'Format': u'%s/%s' % (s["codec_type"], s.get("codec_name",u'?')), - 'Rate': rate or u'-', - 'Dimensions': dim or (None, None), - }) - - if 'image' not in d["format"]["format_name"]: - self._info['Streams'] = result - return - def _util_merge_Regions(self, regs, sub=False, overlap=False, close=False): # sub=False, overlap=False, close=False ; level 0 ; similar regions, similar position (default) # sub=True, overlap=False, close=False ; level 1 ; region contained in other, any shape/size @@ -2410,70 +2467,74 @@ return (regs, drop) - def _detect_AudioFeatures_MUSIC21(self): - # skip file formats not supported - if (self.image_mime[1] not in ['midi']): - return - self._info['Audio'] = [] +class PngFile(JpegFile): + pass - import _music21 as music21 +class GifFile(JpegFile): + pass - #audiofile = '/home/ursin/Desktop/3_Ships.mid' - audiofile = self.image_path +class TiffFile(JpegFile): + pass - #music21.features.jSymbolic.getCompletionStats() - try: - #s = music21.converter.ConverterMidi() - #s.parseFile(audiofile) - #s = s.stream - s = music21.midi.translate.midiFilePathToStream(audiofile) - except music21.midi.base.MidiException: - pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') +class XcfFile(JpegFile): + pass + +class SvgFile(JpegFile): + pass + +class PdfFile(JpegFile): + pass + +#class DjvuFile(JpegFile): +# pass + + +class OggFile(JpegFile): + def getFeatures(self): + # general handling of all audio and video formats + self._detect_Streams_FFMPEG() + + # general audio feature extraction +# self._detect_AudioFeatures_YAAFE() + + def _detect_Streams_FFMPEG(self): + self._info['Streams'] = [] + + # skip file formats that interfere and can cause strange results (pdf is oga?!) + # or file formats not supported (yet?) + if (self.image_mime[1] in ['pdf']) or (self.image_fileext in [u'.svg']): return - #fs = music21.features.jSymbolic.extractorsById - #for k in fs: - # for i in range(len(fs[k])): - # if fs[k][i] is not None: - # n = fs[k][i].__name__ - # if fs[k][i] not in music21.features.jSymbolic.featureExtractors: - # n += " (not implemented)" - # print k, i, n - # else: - # fe = fs[k][i](s) - # print k, i, n, - # try: - # f = fe.extract() - # print f.name, f.vector - # except AttributeError: - # print "ERROR" - data = {} - for item in ['MostCommonPitchFeature', - 'ImportanceOfBassRegisterFeature', - 'ImportanceOfMiddleRegisterFeature', - 'ImportanceOfHighRegisterFeature', - 'AverageNoteDurationFeature', - 'MaximumNoteDurationFeature', - #'DurationFeature', - 'InitialTempoFeature', - 'MaximumNumberOfIndependentVoicesFeature', - 'AverageNumberOfIndependentVoicesFeature',]: - fe = getattr(music21.features.jSymbolic, item)(s) - f = fe.extract() - #data[f.name] = f.vector[0] - data[item.replace('Feature', '')] = f.vector[0] - #print s.duration - data['Duration'] = s.highestTime - #print s.offsetMap - #print s.measureOffsetMap() - data['Metadata'] = s.metadata - data['Lyrics'] = s.lyrics() - #print s.seconds - #print s.secondsMap + # audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + if not d: + return - self._info['Audio'] = [data] -# print self._info['Audio'] + result = [] + for s in d['streams']: + #print s + if (s["codec_type"] == "video"): + rate = s["avg_frame_rate"] + dim = (int(s["width"]), int(s["height"])) + #asp = s["display_aspect_ratio"] + elif (s["codec_type"] == "audio"): +# switch this part off since 'ffprobe' (on toolserver) is too old +# rate = u'%s/%s/%s' % (s["channels"], s["sample_fmt"], s["sample_rate"]) + rate = u'%s/%s/%s' % (s["channels"], u'-', int(float(s["sample_rate"]))) + dim = None + elif (s["codec_type"] == "data"): + rate = None + dim = None + + result.append({ 'ID': int(s["index"]) + 1, + 'Format': u'%s/%s' % (s["codec_type"], s.get("codec_name",u'?')), + 'Rate': rate or u'-', + 'Dimensions': dim or (None, None), + }) + + if 'image' not in d["format"]["format_name"]: + self._info['Streams'] = result return def _detect_AudioFeatures_YAAFE(self): @@ -2634,10 +2695,125 @@ return +class MidiFile(UnknownFile): + def getProperties(self): + result = {} + result.update( self._detect_HeaderAndMetadata() ) # Metadata + result.update( self._detect_Properties_PIL() ) # Properties + return result + + def getFeatures(self): + result = {} + result.update( self._detect_AudioFeatures_MUSIC21() ) # Audio + return result + + def _detect_HeaderAndMetadata(self): + result = {} + + # extract data from midi file + # http://valentin.dasdeck.com/midi/midifile.htm + # http://stackoverflow.com/questions/3943149/reading-and-interpreting-data-fr… + ba = bytearray(open(self.filename, 'rb').read()) + i = -1 + for key, data in [('Text', '\x01'), ('Copyright', '\x02'), ('Lyrics', '\x05')]: + result[key] = [] + while True: + i = ba.find('\xff%s' % data, i+1) + if i < 0: # something found? + break + e = (i+3+ba[i+2]) + if ba[e] != 0: # length match with string end (00)? + e = ba.find('\x00', (i+3+ba[i+2])) + result[key].append(ba[i+3:e].decode('latin-1').strip()) + result[key] = u'\n'.join(result[key]) + + import _music21 as music21 + try: + s = music21.converter.parse(self.filename) + if s.metadata: + pywikibot.output(unicode(s.metadata)) + result['Metadata'].update(s.metadata) + except music21.midi.base.MidiException: + pass + + self._info['Metadata'] = [result] + return {'Metadata': [result]} + + # midi audio feature extraction + def _detect_AudioFeatures_MUSIC21(self): + # skip file formats not supported + if (self.image_mime[1] not in ['midi']): + return + + import _music21 as music21 + + #audiofile = '/home/ursin/Desktop/3_Ships.mid' + audiofile = self.image_path + + #music21.features.jSymbolic.getCompletionStats() + try: + #s = music21.midi.translate.midiFilePathToStream(audiofile) + s = music21.converter.parse(audiofile) + except music21.midi.base.MidiException: + pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') + return + + #fs = music21.features.jSymbolic.extractorsById + #for k in fs: + # for i in range(len(fs[k])): + # if fs[k][i] is not None: + # n = fs[k][i].__name__ + # if fs[k][i] not in music21.features.jSymbolic.featureExtractors: + # n += " (not implemented)" + # print k, i, n + # else: + # fe = fs[k][i](s) + # print k, i, n, + # try: + # f = fe.extract() + # print f.name, f.vector + # except AttributeError: + # print "ERROR" + data = {'RegisterImportance': (music21.features.jSymbolic.ImportanceOfBassRegisterFeature(s).extract().vector[0], + music21.features.jSymbolic.ImportanceOfMiddleRegisterFeature(s).extract().vector[0], + music21.features.jSymbolic.ImportanceOfHighRegisterFeature(s).extract().vector[0],), + 'NoteDuration': (music21.features.jSymbolic.AverageNoteDurationFeature(s).extract().vector[0], + music21.features.jSymbolic.MaximumNoteDurationFeature(s).extract().vector[0],), + 'IndependentVoices': (music21.features.jSymbolic.AverageNumberOfIndependentVoicesFeature(s).extract().vector[0], + music21.features.jSymbolic.MaximumNumberOfIndependentVoicesFeature(s).extract().vector[0],), + 'MostCommonPitch': music21.features.jSymbolic.MostCommonPitchFeature(s).extract().vector[0], + 'Tempo': music21.features.jSymbolic.InitialTempoFeature(s).extract().vector[0], + 'Duration': s.highestTime, + #'Metadata': s.metadata if s.metadata else u'', + 'Lyrics': s.lyrics(recurse=True) if s.lyrics(recurse=True) else u'',} + #print music21.text.assembleLyrics(s) + #print s.duration + #print s.offsetMap + #print s.measureOffsetMap() + #print s.seconds + #print s.secondsMap + + self._info['Audio'] += [data] + return {'Audio': [data]} + + +FILETYPES = { '*': UnknownFile, + ( 'image', 'jpeg'): JpegFile, + ( 'image', 'png'): PngFile, + ( 'image', 'gif'): GifFile, + ( 'image', 'tiff'): TiffFile, + ( 'image', 'x-xcf'): XcfFile, + ( 'image', 'svg+xml'): SvgFile, + ('application', 'pdf'): PdfFile, +# ( 'image', 'vnd.djvu'): DjvuFile, + ('application', 'ogg'): OggFile, + ( 'audio', 'midi'): MidiFile,} + + # all classification and categorization methods and definitions - default variation # use simplest classification I can think of (self-made) and do categorization # mostly based on filtered/reported features -class CatImages_Default(FileData): +class CatImages_Default(object): #ignore = [] ignore = ['color'] @@ -2854,6 +3030,21 @@ return (u'Graphics', bool(relevance)) +# # Category:MIDI files created with GNU LilyPond +# def _cat_audio_MIDIfilescreatedwithGNULilyPond(self): +# # find metadata in extracted data +# print [item.strip() for item in re.findall('Generated .*?\n', u'\n'.join(result['Text']))] +# #u"Cr'eateur: GNU LilyPond 2.0.1" +# import dateutil.parser +# dates = [] +# for line in result['Text']: +# # http://stackoverflow.com/questions/3276180/extracting-date-from-a-string-in… +# try: +# dates.append(dateutil.parser.parse(line, fuzzy=True).isoformat(' ').decode('utf-8')) +# except ValueError: +# pass +# print dates + # Category:Categorized by DrTrigonBot def _addcat_BOT(self): # - ALWAYS - @@ -3099,7 +3290,7 @@ self.image_mime = re.split('[/;\s]', m.file(self.image_path)) #self.image_size = (None, None) mime = mimetypes.guess_all_extensions('%s/%s' % tuple(self.image_mime[0:2])) - if self.image_fileext.lower() not in mime: + if mime and (self.image_fileext.lower() not in mime): pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) @@ -3146,6 +3337,9 @@ #data = Popen("identify -verbose info: %s" % self.image_path, # shell=True, stderr=PIPE).stderr.read() #print data + if not os.path.exists(self.image_path_JPEG): + # xcf can have more than 1 layer/page like gif, tiff, and movies... + self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') self.image_size = Image.open(self.image_path_JPEG).size else: try: @@ -3168,6 +3362,8 @@ self.image_size = (img.shape[1], img.shape[0]) except: + if os.path.exists(self.image_path_JPEG): + os.remove(self.image_path_JPEG) self.image_path_JPEG = self.image_path except: self.image_path_JPEG = self.image_path @@ -3502,69 +3698,38 @@ # gather data from all information interfaces def gatherFeatures(self): - # Image size - self._detect_Properties_PIL() - + self._info['Properties'] = [{'Format': u'-', 'Pages': 0}] + self._info['Metadata'] = [] + self._info['ColorAverage'] = [] + self._info['ColorRegions'] = [] self._info['Faces'] = [] - # Faces (extract EXIF data) - self._detect_Faces_EXIF() - # Faces and eyes (opencv pre-trained haar) - self._detect_Faces_CV() -# TODO: test and use or switch off - # Face via Landmark(s) -# self._detect_FaceLandmark_xBOB() - # exclude duplicates (CV and EXIF) - faces = [item['Position'] for item in self._info['Faces']] - for i in self._util_merge_Regions(faces)[1]: - del self._info['Faces'][i] + self._info['OpticalCodes'] = [] + self._info['People'] = [] + self._info['Chessboard'] = [] + self._info['Text'] = [] + self._info['Streams'] = [] + self._info['Audio'] = [] + self._info['Legs'] = [] + self._info['Hands'] = [] + self._info['Torsos'] = [] + self._info['Ears'] = [] + self._info['Eyes'] = [] + self._info['Automobiles'] = [] - # Segments and colors - self._detect_SegmentColors_JSEGnPIL() - # Average color - self._detect_AverageColor_PILnCV() + # split detection and extraction according to file types; JpegFile, ... + TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) + tf = TypeFile(self.image_path) + import copy + tf.__dict__.update(copy.deepcopy(self.__dict__)) + for func in ['getProperties', 'getFeatures']: + result = getattr(tf, func)() + if result: + self._info.update(result) +# print self._info +# print tf._info + #print tf.__dict__ + self._info = tf._info - # People/Pedestrian (opencv pre-trained hog and haarcascade) - self._detect_People_CV() - - # Geometric object (opencv hough line, circle, edges, corner, ...) - self._detect_Geometry_CV() - - # general (opencv pre-trained, third-party and self-trained haar - # and cascade) classification - # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… - for cf in self.cascade_files: - self._detect_Trained_CV(*cf) - - # optical and other text recognition (tesseract & ocropus, ...) - self._detect_EmbeddedText_poppler() -# self._recognize_OpticalText_ocropus() - # (may be just classify as 'contains text', may be store text, e.g. to wikisource) - - # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) - self._recognize_OpticalCodes_dmtxNzbar() - - # Chessboard (opencv reference detector) - self._detect_Chessboard_CV() - - # general (self-trained) detection WITH classification - # BoW: uses feature detection (SIFT, SURF, ...) AND classification (SVM, ...) -# self._detectclassify_ObjectAll_CV() - # Wavelet: uses wavelet transformation AND classification (machine learning) -# self._detectclassify_ObjectAll_PYWT() - - # general handling of all audio and video formats - self._detect_Streams_FFMPEG() - - # general file EXIF history information - self._detect_History_EXIF() - - # general audio feature extraction -# self._detect_AudioFeatures_YAAFE() - - # midi audio feature extraction -# TODO: improve midi/audio stuff ... - self._detect_AudioFeatures_MUSIC21() - def _existInformation(self, info, ignore = ['Properties', 'ColorAverage']): result = [] for item in info: @@ -3684,6 +3849,11 @@ result = self._info['Streams'] return {'Streams': result} +# def _filter_Audio(self): +# # use all, (should be reliable) +# result = self._info['Audio'] +# return {'Audio': result} + #def _filter_Geometry(self): # result = [] # for item in self._info['Geometry']:

1 0

SVN: [11620] branches/rewrite/pywikibot/userinterfaces
by valhallasw＠svn.wikimedia.org 07 Jun '13

07 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11620 Revision: 11620 Author: valhallasw Date: 2013-06-06 21:19:06 +0000 (Thu, 06 Jun 2013) Log Message: ----------- Removed disfunctional interfaces from rewrite Removed Paths: ------------- branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py Deleted: branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py =================================================================== --- branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py 2013-06-06 19:38:32 UTC (rev 11619) +++ branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py 2013-06-06 21:19:06 UTC (rev 11620) @@ -1,229 +0,0 @@ -# -*- coding: utf-8 -*- -# -# (C) Pywikipedia bot team, 2005-2008 -# -# Distributed under the terms of the MIT license. -# -__version__ = '$Id$' - -import re -import sys -import threading -import time -import tkMessageBox, tkSimpleDialog -from Tkinter import * -from gui import EditBoxWindow - -color_pattern = re.compile(r"%s\{(?P<colorname>\w+)\}" % "\x03") - -# we run the Tkinter mainloop in a separate thread so as not to block -# the main bot code; however, this means that all communication with -# the Tkinter interface has to be done through events that will be processed -# by the mainloop in the separate thread. Code outside this module must not -# call any of the Tkinter objects directly. - - -class MainloopThread(threading.Thread): - def __init__(self, window): - threading.Thread.__init__(self) - self.window = window - - def run(self): - self.window.mainloop() - - -class CustomMessageBox(tkSimpleDialog.Dialog): - def __init__(self, master, question, options, hotkeys, default=None): - self.question = question - self.options = options - self.hotkeys = hotkeys - self.default = default - tkSimpleDialog.Dialog.__init__(self, master) - - def body(self, master): - Label(self, text=self.question).grid(columnspan = len(self.options)) - btns = [] - for i in xrange(len(self.options)): - # mark hotkey with underline - m = re.search('[%s%s]' % (self.hotkeys[i].lower(), - self.hotkeys[i].upper()), self.options[i]) - if m: - pos = m.start() - else: - self.options[i] += ' (%s)' % self.hotkeys[i] - pos = len(self.options[i]) - 2 - b = Button(self, text=self.options[i], - underline=pos, - command=lambda h=self.hotkeys[i]: self.select(h)) - self.bind("<Control-%s>" % self.hotkeys[i], - lambda h=self.hotkeys[i]: self.select(h)) - b.grid(row = 1, column = i) - btns.append(b) - if self.default and self.default in self.hotkeys: - return btns[self.hotkeys.index(self.default)] - else: - return btns[0] - - def buttonbox(self): - return - - def select(self, i, event=None): - self.selection = i - self.ok() - - def apply(self): - if self.default and not self.selection: - self.selection = self.default - - -class OutputBox(Text): - def __init__(self, parent, *args, **kwargs): - Text.__init__(self, parent, *args, **kwargs) - # create color tags - # map Unix/ANSI color names to Tcl/Tk names - # because Tkinter background is white, we need darker colors - for ucolor, tcolor in ( - ('default', 'Black'), - ('lightblue', 'Blue'), - ('lightgreen', 'Green'), - ('lightaqua', 'DarkSeaGreen'), - ('lightred', 'Red'), - ('lightpurple', 'DarkViolet'), - ('lightyellow', 'DarkOrange') - ): - self.tag_config(ucolor, foreground=tcolor) - - def show(self, text): - global debugger - debugger = text - next_tag = 'default' - m = color_pattern.search(text) - while m: - self.insert(END, text[0:m.start()], next_tag) - next_tag = m.group("colorname") - text = text[m.end():] - m = color_pattern.search(text) - self.insert(END, text, next_tag) - self.yview(END) - - -class TkController(Frame): - """ - Tkinter user interface controller. - - This object receives and processes events dispatched to it by the UI - object. Do not call this object's methods directly. Methods of this - object cannot return values to the main thread; they must store them - in attributes to be retrieved by the main thread. - - """ - # TODO: use Event for inter-thread communication instead of wait loops - def __init__(self, parent, **kwargs): - Frame.__init__(self, parent, **kwargs) - - def showinfo(self, text, wait=True): - """ - Show a pop-up message. - - Set wait to False to allow the pop-up to remain on screen while the - bot continues to work. - - """ - box = tkMessageBox.showinfo("Bot Message", text) - box.display() - if wait: - self.wait_window(box.top) - - def ask(self, question, password=False): - """ - Show a question in a dialog window and store the user's answer. - """ - self.answer = tkSimpleDialog.askstring('Question', question) - - def ask_choice(self, question, options, hotkeys, default): - d = CustomMessageBox(self, question, options, hotkeys) - self.wait_window(d.top) - self.selection = d.selection or d.default - - def edit_text(self, text, jumpIndex, highlight): - editBoxWindow = EditBoxWindow(text) - editBoxWindow.highlight(highlight) - self.wait_window(editBoxWindow.top) - self.edited_text = editBoxWindow.text - - -class UI(object): - """ - Tkinter user interface. - - This object serves only to dispatch event calls to the TkController - object, to be run in that object's separate mainloop thread; and, - when necessary, to wait for the user's response. - - """ - def __init__(self, parent = None): - # create a new window if necessary - self.control = TkController(parent or Tk()) - - # textarea with vertical scrollbar - scrollbar = Scrollbar(self.control) - self.logBox = OutputBox(self.control, yscrollcommand=scrollbar.set) - - # add scrollbar to main frame, associate it with our editbox - scrollbar.pack(side=RIGHT, fill=Y) - scrollbar.config(command=self.logBox.yview) - - # put textarea into top frame, using all available space - self.logBox.pack(anchor=CENTER, fill=BOTH) - self.control.pack(side=TOP) - - MainloopThread(self.control).start() - - def output(self, text, urgency=1, toStdout=False, wait=True): - """ - urgency levels: (NOT IMPLEMENTED) - 0 - Debug output. Won't be shown in normal mode. - 1 - Will be shown in log window. - 2 - Will be shown in error box. - - TODO: introduce constants - - wait: block bot until user dismisses pop-up window (level 2 only) - """ - if urgency >= 2: - self.control.after_idle(self.control.showinfo, text, wait) - elif urgency >= 1: - self.control.after_idle(self.logBox.show, text) - - def input(self, question, password = False): - """ - Returns a unicode string. - """ - # TODO: hide input if password = True - self.control.after_idle(self.control.ask, question) - # wait until the answer has been given - while not hasattr(self.control, "answer"): - time.sleep(1) - # answer needs to be deleted so that it won't be reused the - # next time this method is called - answer = self.control.answer - del self.control.answer - return answer - - def editText(self, text, jumpIndex = None, highlight = None): - self.control.after_idle(self.control.edit_text, - text, jumpIndex, highlight) - while not hasattr(self.control, "edited_text"): - time.sleep(1) - result = self.control.edited_text - del self.control.edited_text - return result - - def inputChoice(self, question, options, hotkeys, default = None): - self.control.after_idle(self.control.ask_choice, question, - options, hotkeys, default) - while not hasattr(self.control, "selection"): - time.sleep(1) - selection = self.control.selection - del self.control.selection - return selection Deleted: branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py =================================================================== --- branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py 2013-06-06 19:38:32 UTC (rev 11619) +++ branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py 2013-06-06 21:19:06 UTC (rev 11620) @@ -1,64 +0,0 @@ - -__version__ = '$Id$' - -import sys; sys.path.append('..') - -import re -import terminal_interface -import wx - -app = wx.App() - -class UI(terminal_interface.UI): - def __init__(self): - pass - - def input(self, question, password = False): - """ - Works like raw_input(), but returns a unicode string instead of ASCII. - - Unlike raw_input, this function automatically adds a space after the - question. - """ - # TODO: hide input if password = True - - self.output(question) - if password: - answer = wx.PasswordEntryDialog( None, question, '','') - else: - answer = wx.TextEntryDialog( None, question, '', '' ) - answer.ShowModal() - self.output(answer+'\n') - #tkSimpleDialog.askstring('title', question) - return answer.GetValue() or '' - - def inputChoice(self, question, options, hotkeys, default = None): - for i in range(len(options)): - option = options[i] - hotkey = hotkeys[i] - m = re.search('[%s%s]' % (hotkey.lower(), hotkey.upper()), option) - if m: - pos = m.start() - options[i] = '%s[%s]%s' % (option[:pos], option[pos], option[pos+1:]) - else: - options[i] = '%s [%s]' % (option, hotkey) - - while True: - prompt = '%s\n(%s)' % (question, ', '.join(options)) - self.output('%s (%s)' % (question, ', '.join(options))) - answer = wx.TextEntryDialog(None, prompt, question, '') - answer.ShowModal() - answer = answer.GetValue() - self.output(answer+'\n') - - if answer.lower() in hotkeys or answer.upper() in hotkeys: - return answer - elif default and answer=='':# empty string entered - return default - -if __name__ == '__main__': - ui = UI() - print ui.input('Test?') - -app.MainLoop() -

1 0

SVN: [11619] branches/rewrite/pywikibot/site.py
by legoktm＠svn.wikimedia.org 06 Jun '13

06 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11619 Revision: 11619 Author: legoktm Date: 2013-06-06 19:38:32 +0000 (Thu, 06 Jun 2013) Log Message: ----------- Only add patrol flag if showPatrolled is not None, user can have either patrol or patrolmarks right. Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2013-06-02 17:01:35 UTC (rev 11618) +++ branches/rewrite/pywikibot/site.py 2013-06-06 19:38:32 UTC (rev 11619) @@ -2229,7 +2229,7 @@ 'anon': showAnon, 'redirect': showRedirects, } - if self.has_right('patrol'): + if showPatrolled is not None and (self.has_right('patrol') or self.has_right('patrolmarks')): rcgen.request['rcprop'] += '|patrolled' filters['patrolled'] = showPatrolled rcshow = []

1 0

SVN: [11618] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11618 Revision: 11618 Author: drtrigon Date: 2013-06-02 17:01:35 +0000 (Sun, 02 Jun 2013) Log Message: ----------- improvement; POSIT refactored into own method, does projection to calc errors also Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-02 13:07:05 UTC (rev 11617) +++ trunk/pywikipedia/catimages.py 2013-06-02 17:01:35 UTC (rev 11618) @@ -405,7 +405,7 @@ D2points = [ item[:2] + item[2:]/2. for item in D2points ] neutral = np.array([[np.pi],[0.],[0.]]) # calculate pose - rvec, tvec, cm, err = self._util_get_Pose(D3points, D2points, self.image_size) + rvec, tvec, cm, err = self._util_get_Pose_solvePnP(D3points, D2points, self.image_size) #data['Pose'] = tuple(rvec[:,0]) check = not (err[:,0,:].max() > 0.5) if not check: @@ -431,18 +431,18 @@ # drv, np.zeros((3,1)))[0] #print (rvec - nrv < 1E-12) # compare data['Pose'] = map(float, tuple(drv[:,0])) -# TODO: POSIT has to be tested and compared to solvePnP; which one is better?? +# TODO: POSIT has to be tested and compared; draw both results! # POSIT: http://www.cfar.umd.edu/~daniel/daniel_papersfordownload/Pose25Lines.pdf if False: pywikibot.output("solvePnP:") pywikibot.output(str(rvec[:,0])) pywikibot.output(str(tvec[:,0])) - import opencv - #opencv.unit_test() - (rmat, tvec, mdl) = opencv.posit(D3points, D2points, (100, 1.0e-4)) + pywikibot.output(str(err[:,0,:])) + rvec, tvec, cm, err = self._util_get_Pose_POSIT(D3points, D2points) pywikibot.output("POSIT:") - pywikibot.output(str(cv2.Rodrigues(rmat)[0][:,0])) + pywikibot.output(str(rvec[:,0])) pywikibot.output(str(tvec)) + pywikibot.output(str(np.array(err)[:,0,:]/max(self.image_size))) result.append( data ) ## see '_drawRect' @@ -455,7 +455,7 @@ self._info['Faces'] += result return - def _util_get_Pose(self, D3points, D2points, shape): + def _util_get_Pose_solvePnP(self, D3points, D2points, shape): """ Calculate pose from head model "little girl" w/o camera or other calibrations needed. @@ -488,6 +488,41 @@ return rvec, tvec, np.array(cameraMatrix), (np.array(err)/max_d) + #def _util_get_Pose_POSIT(self, D3points, D2points, shape): + def _util_get_Pose_POSIT(self, D3points, D2points): + """ Calculate pose from head model "little girl" w/o camera or other + calibrations needed. + + Method similar to '_util_get_Pose_solvePnP', please compare. + + D2points: left eye, right eye, nose, mouth + """ + # calculate pose + import opencv + #opencv.unit_test() + (rmat, tvec, mdl) = opencv.posit(D3points, D2points, (100, 1.0e-4)) + rvec = cv2.Rodrigues(rmat)[0] + + # Project the model points with the estimated pose + # http://opencv.willowgarage.com/documentation/cpp/camera_calibration_and_3d_… + # intrinsic: camera matrix + # extrinsic: rotation-translation matrix [R|t] + # CV_32F, principal point in the centre of the image is (0, 0) instead of (self.image_size[0]*0.5) + FOCAL_LENGTH = 760.0 # hard-coded in posit_python.cpp, should be changed... + cameraMatrix = [[FOCAL_LENGTH, 0.0, 0.0],#shape[0]*0.0], + [ 0.0, FOCAL_LENGTH, 0.0],#shape[1]*0.0], + [ 0.0, 0.0, 1.0],] + + # compare to 2D points + err = [] + for i, vec in enumerate(np.array(mdl)): + nvec = np.dot(cameraMatrix, (np.dot(rmat, vec) + tvec)) + err.append(((D2points[i] - nvec[:2]/nvec[2]), D2points[i], nvec[:2]/nvec[2])) + + #pywikibot.output(u'result for UN-calibrated camera:\n rot=%s' % rvec.transpose()[0]) + + return rvec, tvec, np.array(cameraMatrix), (np.array(err)/1.0) + # https://pypi.python.org/pypi/xbob.flandmark # http://cmp.felk.cvut.cz/~uricamic/flandmark/ def _detect_FaceLandmark_xBOB(self): @@ -3849,9 +3884,9 @@ #outresult = [ outpage.get() ] + outresult # append to page outresult = u"\n".join(outresult) pywikibot.output(u"Size of log page data: %s byte(s)" % len(outresult)) - # write pages mutliple times if content is too large in order to circumvent: + # work-a-round: write pages mutliple times if content is too large in order to circumvent # "HTTPError: 504 Gateway Time-out" leading finally to "MaxTriesExceededError" - # (why is that...?!?? FIX THIS!) + # (why is that...?!?? FIX THIS in the framework core e.g. 'postForm'!) tmp = outresult while tmp: i = np.array([m.start() for m in re.finditer(u"\n\n==", tmp)]

1 0

SVN: [11617] branches/rewrite/tests/page_tests.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11617 Revision: 11617 Author: legoktm Date: 2013-06-02 13:07:05 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Add tests for page.get() Modified Paths: -------------- branches/rewrite/tests/page_tests.py Modified: branches/rewrite/tests/page_tests.py =================================================================== --- branches/rewrite/tests/page_tests.py 2013-06-02 12:49:02 UTC (rev 11616) +++ branches/rewrite/tests/page_tests.py 2013-06-02 13:07:05 UTC (rev 11617) @@ -215,6 +215,16 @@ self.assertTrue(p1.isRedirectPage()) self.assertEqual(p1.getRedirectTarget(), p2) + def testPageGet(self): + p1 = pywikibot.Page(site, u'User:Legoktm/R2') + p2 = pywikibot.Page(site, u'User:Legoktm/R1') + p3 = pywikibot.Page(site, u'User:Legoktm/R3') + + text = u'This page is used in the [[mw:Manual:Pywikipediabot]] testing suite.' + self.assertEqual(p1.get(), text) + self.assertRaises(pywikibot.exceptions.IsRedirectPage, p2.get) + self.assertRaises(pywikibot.exceptions.NoPage, p3.get) + def testApiMethods(self): """Test various methods that rely on API.""" # since there is no way to predict what data the wiki will return, @@ -307,7 +317,6 @@ ## def autoFormat(self): ## def isAutoTitle(self): -## def get(self, force=False, get_redirect=False, sysop=False): ## def getOldVersion(self, oldid, force=False, get_redirect=False, ## sysop=False): ## text = property(_textgetter, _textsetter, _cleartext,

1 0

SVN: [11616] branches/rewrite/tests/page_tests.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11616 Revision: 11616 Author: legoktm Date: 2013-06-02 12:49:02 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Add test for page.getRedirectTarget() Modified Paths: -------------- branches/rewrite/tests/page_tests.py Modified: branches/rewrite/tests/page_tests.py =================================================================== --- branches/rewrite/tests/page_tests.py 2013-06-02 12:15:06 UTC (rev 11615) +++ branches/rewrite/tests/page_tests.py 2013-06-02 12:49:02 UTC (rev 11616) @@ -209,6 +209,12 @@ self.assertEqual(p2.isImage(), True) self.assertEqual(p3.isImage(), False) + def testIsRedirect(self): + p1 = pywikibot.Page(site, u'User:Legoktm/R1') + p2 = pywikibot.Page(site, u'User:Legoktm/R2') + self.assertTrue(p1.isRedirectPage()) + self.assertEqual(p1.getRedirectTarget(), p2) + def testApiMethods(self): """Test various methods that rely on API.""" # since there is no way to predict what data the wiki will return, @@ -312,7 +318,6 @@ ## def backlinks(self, followRedirects=True, filterRedirects=None, ## namespaces=None): ## def embeddedin(self, filter_redirects=None, namespaces=None): -## def getRedirectTarget(self): ## def getVersionHistory(self, reverseOrder=False, getAll=False, ## revCount=500): ## def getVersionHistoryTable(self, forceReload=False, reverseOrder=False,

1 0

SVN: [11615] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11615 Revision: 11615 Author: drtrigon Date: 2013-06-02 12:15:06 +0000 (Sun, 02 Jun 2013) Log Message: ----------- bug fix; work-a-round for MaxTriesExceededError(s) caused due to to big page content write attempt (should be solved properly in framework core, e.g. 'postData') Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-02 12:06:55 UTC (rev 11614) +++ trunk/pywikipedia/catimages.py 2013-06-02 12:15:06 UTC (rev 11615) @@ -432,6 +432,7 @@ #print (rvec - nrv < 1E-12) # compare data['Pose'] = map(float, tuple(drv[:,0])) # TODO: POSIT has to be tested and compared to solvePnP; which one is better?? + # POSIT: http://www.cfar.umd.edu/~daniel/daniel_papersfordownload/Pose25Lines.pdf if False: pywikibot.output("solvePnP:") pywikibot.output(str(rvec[:,0])) @@ -3474,6 +3475,7 @@ self._detect_Faces_EXIF() # Faces and eyes (opencv pre-trained haar) self._detect_Faces_CV() +# TODO: test and use or switch off # Face via Landmark(s) # self._detect_FaceLandmark_xBOB() # exclude duplicates (CV and EXIF) @@ -3507,7 +3509,6 @@ self._recognize_OpticalCodes_dmtxNzbar() # Chessboard (opencv reference detector) -# TODO: apply pose estimation/detection to faces (eyes, mouth, ... or landmark) also self._detect_Chessboard_CV() # general (self-trained) detection WITH classification @@ -3846,13 +3847,26 @@ if outresult: outpage = pywikibot.Page(site, u"User:DrTrigon/User:DrTrigonBot/logging") #outresult = [ outpage.get() ] + outresult # append to page - outpage.put( u"\n".join(outresult), comment="bot writing log for last run" ) + outresult = u"\n".join(outresult) + pywikibot.output(u"Size of log page data: %s byte(s)" % len(outresult)) + # write pages mutliple times if content is too large in order to circumvent: + # "HTTPError: 504 Gateway Time-out" leading finally to "MaxTriesExceededError" + # (why is that...?!?? FIX THIS!) + tmp = outresult + while tmp: + i = np.array([m.start() for m in re.finditer(u"\n\n==", tmp)] + + [len(tmp)]) + #pos = i[ np.where((i - 2048*1024) <= 0)[0][-1] ] # $wgMaxArticleSize + pos = i[ np.where((i - 500*1024) <= 0)[0][-1] ] + pywikibot.output(u"Size of bunch to write: %s byte(s)" % len(tmp[:pos])) + outpage.put( tmp[:pos], comment="bot writing log for last run" ) + tmp = tmp[pos:] if pywikibot.simulate: #print u"--- " * 20 #print u"--- " * 20 - #print u"\n".join(outresult) + #print outresult posfile = open(os.path.join(scriptdir, 'cache/catimages.log'), "a") - posfile.write( u"\n".join(outresult) ) + posfile.write( outresult ) posfile.close() # http://scipy-lectures.github.com/advanced/scikit-learn/index.html

1 0

SVN: [11614] branches/rewrite/pywikibot/data/api.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11614 Revision: 11614 Author: legoktm Date: 2013-06-02 12:06:55 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Try to use cPickle if possible for a speedboost Modified Paths: -------------- branches/rewrite/pywikibot/data/api.py Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2013-06-02 12:02:35 UTC (rev 11613) +++ branches/rewrite/pywikibot/data/api.py 2013-06-02 12:06:55 UTC (rev 11614) @@ -19,7 +19,10 @@ import logging import mimetypes import os -import pickle +try: + import cPickle as pickle +except ImportError: + import pickle import pprint import re import traceback

1 0

SVN: [11613] branches/rewrite/tests/api_tests.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11613 Revision: 11613 Author: legoktm Date: 2013-06-02 12:02:35 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Add a unit test that verifies the cached data is read correctly Modified Paths: -------------- branches/rewrite/tests/api_tests.py Modified: branches/rewrite/tests/api_tests.py =================================================================== --- branches/rewrite/tests/api_tests.py 2013-06-02 11:40:03 UTC (rev 11612) +++ branches/rewrite/tests/api_tests.py 2013-06-02 12:02:35 UTC (rev 11613) @@ -6,6 +6,7 @@ # __version__ = '$Id$' +import datetime import unittest import pywikibot import pywikibot.data.api as api @@ -71,6 +72,22 @@ self.assert_(page.title() in titles) +class TestCachedRequest(unittest.TestCase): + def testResults(self): + # Run the cached query twice to ensure the + # data returned is equal + params = {'action': 'query', + 'prop': 'info', + 'titles': 'Main Page', + } + req = api.CachedRequest(datetime.timedelta(minutes=10), + site=mysite, **params) + data = req.submit() + req2 = api.CachedRequest(datetime.timedelta(minutes=10), + site=mysite, **params) + data2 = req2.submit() + self.assertEqual(data, data2) + if __name__ == '__main__': try: unittest.main()

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn