Pywikipedia-svn June 2013

pywikipedia-svn@lists.wikimedia.org

8 participants
103 discussions

SVN: [11624] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 08 Jun '13

08 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11624 Revision: 11624 Author: drtrigon Date: 2013-06-07 23:51:26 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; most methods renamed, wrapper GenericFile introduced Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-07 22:29:18 UTC (rev 11623) +++ trunk/pywikipedia/catimages.py 2013-06-07 23:51:26 UTC (rev 11624) @@ -144,9 +144,10 @@ # all detection and recognition methods - bindings to other classes, modules and libs -class UnknownFile(object): - def __init__(self, filename, *args, **kwargs): - self.filename = filename +class _UnknownFile(object): + def __init__(self, file_name, file_mime, *args, **kwargs): + self.file_name = file_name + self.file_mime = file_mime self.image_size = (None, None) # available file properties and metadata @@ -183,12 +184,157 @@ return self._properties def getFeatures(self): - pywikibot.warning(u"File format '%s/%s' not supported (yet)!" % tuple(self.image_mime[:2])) + pywikibot.warning(u"File format '%s/%s' not supported (yet)!" % tuple(self.file_mime[:2])) return self._features def _detect_HeaderAndMetadata(self): # check/look into the file by midnight commander (mc) # https://pypi.python.org/pypi/hachoir-metadata + +#### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +#try: +# from hachoir_core.error import error, HachoirError +# from hachoir_core.cmd_line import unicodeFilename +# from hachoir_core.i18n import getTerminalCharset, _ +# from hachoir_core.benchmark import Benchmark +# from hachoir_core.stream import InputStreamError +# from hachoir_core.tools import makePrintable +# from hachoir_parser import createParser, ParserList +# import hachoir_core.config as hachoir_config +# from hachoir_metadata import config +#except ImportError, err: +# raise +# print >>sys.stderr, "Unable to import an Hachoir module: %s" % err +# sys.exit(1) +#from optparse import OptionGroup, OptionParser +#from hachoir_metadata import extractMetadata +#from hachoir_metadata.metadata import extractors as metadata_extractors +# +# +#def parseOptions(): +# parser = OptionParser(usage="%prog [options] files") +# parser.add_option("--type", help=_("Only display file type (description)"), +# action="store_true", default=False) +# parser.add_option("--mime", help=_("Only display MIME type"), +# action="store_true", default=False) +# parser.add_option("--level", +# help=_("Quantity of information to display from 1 to 9 (9 is the maximum)"), +# action="store", default="9", type="choice", +# choices=[ str(choice) for choice in xrange(1,9+1) ]) +# parser.add_option("--raw", help=_("Raw output"), +# action="store_true", default=False) +# parser.add_option("--bench", help=_("Run benchmark"), +# action="store_true", default=False) +# parser.add_option("--force-parser",help=_("List all parsers then exit"), +# type="str") +# parser.add_option("--profiler", help=_("Run profiler"), +# action="store_true", default=False) +# parser.add_option("--quality", help=_("Information quality (0.0=fastest, 1.0=best, and default is 0.5)"), +# action="store", type="float", default="0.5") +# parser.add_option("--maxlen", help=_("Maximum string length in characters, 0 means unlimited (default: %s)" % config.MAX_STR_LENGTH), +# type="int", default=config.MAX_STR_LENGTH) +# parser.add_option("--verbose", help=_("Verbose mode"), +# default=False, action="store_true") +# parser.add_option("--debug", help=_("Debug mode"), +# default=False, action="store_true") +# +# values, filename = parser.parse_args() +# if len(filename) == 0: +# parser.print_help() +# sys.exit(1) +# +# # Update limits +# config.MAX_STR_LENGTH = values.maxlen +# if values.raw: +# config.RAW_OUTPUT = True +# +# return values, filename +# +#def processFile(values, filename, +#display_filename=False, priority=None, human=True, display=True): +# charset = getTerminalCharset() +# filename, real_filename = unicodeFilename(filename, charset), filename +# +# # Create parser +# try: +# if values.force_parser: +# tags = [ ("id", values.force_parser), None ] +# else: +# tags = None +# parser = createParser(filename, real_filename=real_filename, tags=tags) +# help(parser) +# print parser.getParserTags() +# print parser.PARSER_TAGS +# for i, item in enumerate(parser.createFields()): +# print item +# if i > 5: +# break +# except InputStreamError, err: +# error(unicode(err)) +# return False +# if not parser: +# error(_("Unable to parse file: %s") % filename) +# return False +# +# # Extract metadata +# extract_metadata = not(values.mime or values.type) +# if extract_metadata: +# try: +# metadata = extractMetadata(parser, values.quality) +# except HachoirError, err: +# error(unicode(err)) +# metadata = None +# if not metadata: +# parser.error(_("Hachoir can't extract metadata, but is able to parse: %s") +# % filename) +# return False +# +# if display: +# # Display metadatas on stdout +# if extract_metadata: +# text = metadata.exportPlaintext(priority=priority, human=human) +# if not text: +# text = [_("(no metadata, priority may be too small)")] +# if display_filename: +# for line in text: +# line = "%s: %s" % (filename, line) +# print makePrintable(line, charset) +# else: +# for line in text: +# print makePrintable(line, charset) +# else: +# if values.type: +# text = parser.description +# else: +# text = parser.mime_type +# if display_filename: +# text = "%s: %s" % (filename, text) +# print text +# return True +# +#### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# def processFiles(values, filenames, display=True): +# human = not(values.raw) +# ok = True +# priority = int(values.level)*100 + 99 +# display_filename = (1 < len(filenames)) +# for filename in filenames: +# ok &= processFile(values, filename, display_filename, priority, human, display) +# return ok +# +# try: +# # Parser options and initialize Hachoir +# values, filenames = parseOptions() +# +# ok = processFiles(values, filenames) +# except KeyboardInterrupt: +# print _("Program interrupted (CTRL+C).") +# ok = False +# sys.exit(int(not ok)) +# +#### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + pass def _detect_Properties(self): @@ -196,8 +342,8 @@ pass -class JpegFile(UnknownFile): - # for '_detect_Trained_CV' +class _JpegFile(_UnknownFile): + # for '_detect_Trained' cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), (u'Torsos', 'haarcascade_upperbody.xml'), (u'Ears', 'haarcascade_mcs_leftear.xml'), @@ -212,12 +358,11 @@ # ('Hands' does not behave very well, in fact it detects any kind of skin and other things...) #(u'Aeroplanes', 'haarcascade_aeroplane.xml'),] # e.g. for 'Category:Unidentified aircraft' - def __init__(self, filename, *args, **kwargs): - UnknownFile.__init__(self, filename) + def __init__(self, file_name, file_mime, *args, **kwargs): + _UnknownFile.__init__(self, file_name, file_mime) - self.image_filename = os.path.split(self.filename)[-1] - self.image_fileext = os.path.splitext(self.image_filename)[1] - self.image_path = self.filename + self.image_filename = os.path.split(self.file_name)[-1] + self.image_path = self.file_name self.image_path_JPEG = self.image_path + '.jpg' self._convert() @@ -235,7 +380,7 @@ # Faces (extract EXIF data) self._detect_Faces_EXIF() # Faces and eyes (opencv pre-trained haar) - self._detect_Faces_CV() + self._detect_Faces() # TODO: test and use or switch off # Face via Landmark(s) # self._detect_FaceLandmark_xBOB() @@ -245,36 +390,36 @@ del self._features['Faces'][i] # Segments and colors - self._detect_SegmentColors_JSEGnPIL() + self._detect_SegmentColors() # Average color - self._detect_AverageColor_PILnCV() + self._detect_AverageColor() # People/Pedestrian (opencv pre-trained hog and haarcascade) - self._detect_People_CV() + self._detect_People() # Geometric object (opencv hough line, circle, edges, corner, ...) - self._detect_Geometry_CV() + self._detect_Geometry() # general (opencv pre-trained, third-party and self-trained haar # and cascade) classification # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… for cf in self.cascade_files: - self._detect_Trained_CV(*cf) + self._detect_Trained(*cf) # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) - self._recognize_OpticalCodes_dmtxNzbar() + self._recognize_OpticalCodes() # Chessboard (opencv reference detector) - self._detect_Chessboard_CV() + self._detect_Chessboard() # general (self-trained) detection WITH classification # BoW: uses feature detection (SIFT, SURF, ...) AND classification (SVM, ...) -# self._detectclassify_ObjectAll_CV() +# self._detectclassify_ObjectAll() # Wavelet: uses wavelet transformation AND classification (machine learning) # self._detectclassify_ObjectAll_PYWT() # general file EXIF history information - self._detect_History_EXIF() + self._detect_History() return self._features @@ -320,7 +465,7 @@ try: i = Image.open(self.image_path) except IOError: - pywikibot.warning(u'unknown file type [JpegFile]') + pywikibot.warning(u'unknown file type [_JpegFile]') return # http://mail.python.org/pipermail/image-sig/1999-May/000740.html @@ -351,8 +496,8 @@ 'Palette': str(len(i.palette.palette)) if i.palette else u'-', 'Pages': pc, 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), }) + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), }) #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) @@ -360,7 +505,7 @@ # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html - def _detect_Faces_CV(self): + def _detect_Faces(self): """Converts an image to grayscale and prints the locations of any faces found""" # http://python.pastebin.com/m76db1d6b @@ -429,10 +574,10 @@ # how small and how many features are detected as faces (or eyes) scale = max([1., np.average(np.array(img.shape)[0:2]/500.)]) except IOError: - pywikibot.warning(u'unknown file type [_detect_Faces_CV]') + pywikibot.warning(u'unknown file type [_detect_Faces]') return except AttributeError: - pywikibot.warning(u'unknown file type [_detect_Faces_CV]') + pywikibot.warning(u'unknown file type [_detect_Faces]') return #detectAndDraw( image, cascade, nestedCascade, scale ); @@ -808,7 +953,7 @@ # .../opencv/samples/cpp/peopledetect.cpp # + Haar/Cascade detection - def _detect_People_CV(self): + def _detect_People(self): # http://stackoverflow.com/questions/10231380/graphic-recognition-of-people # https://code.ros.org/trac/opencv/ticket/1298 # http://opencv.itseez.com/modules/gpu/doc/object_detection.html @@ -829,10 +974,10 @@ scale = max([1., np.average(np.array(img.shape)[0:2]/400.)]) #scale = max([1., np.average(np.array(img.shape)[0:2]/300.)]) except IOError: - pywikibot.warning(u'unknown file type [_detect_People_CV]') + pywikibot.warning(u'unknown file type [_detect_People]') return except AttributeError: - pywikibot.warning(u'unknown file type [_detect_People_CV]') + pywikibot.warning(u'unknown file type [_detect_People]') return # similar to face detection @@ -902,7 +1047,7 @@ self._features['People'] = result return - def _detect_Geometry_CV(self): + def _detect_Geometry(self): result = self._util_get_Geometry_CVnSCIPY() self._features['Geometry'] = [{'Lines': result['Lines'], @@ -934,10 +1079,10 @@ # how small and how many features are detected scale = max([1., np.average(np.array(img.shape)[0:2]/500.)]) except IOError: - pywikibot.warning(u'unknown file type [_detect_Geometry_CV]') + pywikibot.warning(u'unknown file type [_detect_Geometry]') return self._buffer_Geometry except AttributeError: - pywikibot.warning(u'unknown file type [_detect_Geometry_CV]') + pywikibot.warning(u'unknown file type [_detect_Geometry]') return self._buffer_Geometry # similar to face or people detection @@ -1065,7 +1210,7 @@ return self._buffer_Geometry # .../opencv/samples/cpp/bagofwords_classification.cpp - def _detectclassify_ObjectAll_CV(self): + def _detectclassify_ObjectAll(self): """Uses the 'The Bag of Words model' for detection and classification""" # CAN ALSO BE USED FOR: TEXT, ... @@ -1165,7 +1310,7 @@ # http://library.wolfram.com/infocenter/Demos/5725/#downloads # http://code.google.com/p/pymeanshift/wiki/Examples # (http://pythonvision.org/basic-tutorial, http://luispedro.org/software/mahotas, http://packages.python.org/pymorph/) - def _detect_SegmentColors_JSEGnPIL(self): # may be SLIC other other too... + def _detect_SegmentColors(self): # may be SLIC other other too... try: #im = Image.open(self.image_path).convert(mode = 'RGB') im = Image.open(self.image_path_JPEG) @@ -1179,7 +1324,7 @@ (l, t) = (0, 0) i = im except IOError: - pywikibot.warning(u'unknown file type [_detect_SegmentColors_JSEGnPIL]') + pywikibot.warning(u'unknown file type [_detect_SegmentColors]') return result = [] @@ -1193,7 +1338,7 @@ ##(pic, scale) = self._util_detect_ColorSegments_JSEG(pic) # (final split) #hist = self._util_get_ColorSegmentsHist_PIL(i, pic, scale) # except TypeError: - pywikibot.warning(u'unknown file type [_detect_SegmentColors_JSEGnPIL]') + pywikibot.warning(u'unknown file type [_detect_SegmentColors]') return i = 0 # (may be do an additional region merge according to same color names...) @@ -1221,14 +1366,14 @@ # http://code.google.com/p/python-colormath/ # http://en.wikipedia.org/wiki/Color_difference # http://www.farb-tabelle.de/en/table-of-color.htm - def _detect_AverageColor_PILnCV(self): + def _detect_AverageColor(self): try: # we need to have 3 channels (but e.g. grayscale 'P' has only 1) #i = Image.open(self.image_path).convert(mode = 'RGB') i = Image.open(self.image_path_JPEG) h = i.histogram() except IOError: - pywikibot.warning(u'unknown file type [_detect_AverageColor_PILnCV]') + pywikibot.warning(u'unknown file type [_detect_AverageColor]') return result = self._util_average_Color_colormath(h) @@ -1321,7 +1466,7 @@ tmpjpg = os.path.join(scriptdir, "cache/jseg_buf.jpg") tmpgif = os.path.join(scriptdir, "cache/jseg_buf.gif") - # same scale func as in '_detect_Faces_CV' + # same scale func as in '_detect_Faces' scale = max([1., np.average(np.array(im.size)[0:2]/200.)]) #print np.array(im.size)/scale, scale try: @@ -1482,7 +1627,7 @@ return im # Category:... (several; look at self.gatherFeatures for more hints) - def _detect_Trained_CV(self, info_desc, cascade_file, maxdim=500.): + def _detect_Trained(self, info_desc, cascade_file, maxdim=500.): # general (self trained) classification (e.g. people, ...) # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… @@ -1511,10 +1656,10 @@ # how small and how many features are detected scale = max([1., np.average(np.array(img.shape)[0:2]/maxdim)]) except IOError: - pywikibot.warning(u'unknown file type [_detect_Trained_CV]') + pywikibot.warning(u'unknown file type [_detect_Trained]') return except AttributeError: - pywikibot.warning(u'unknown file type [_detect_Trained_CV]') + pywikibot.warning(u'unknown file type [_detect_Trained]') return # similar to face detection @@ -1541,7 +1686,7 @@ self._features[info_desc] = result return - def _recognize_OpticalCodes_dmtxNzbar(self): + def _recognize_OpticalCodes(self): # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) # http://libdmtx.wikidot.com/libdmtx-python-wrapper # http://blog.globalstomp.com/2011/09/decoding-qr-code-code-128-code-39.html @@ -1571,7 +1716,7 @@ scale = max([1., np.average(np.array(img.size)/200.)]) except IOError: - pywikibot.warning(u'unknown file type [_recognize_OpticalCodes_dmtxNzbar]') + pywikibot.warning(u'unknown file type [_recognize_OpticalCodes]') return smallImg = img.resize( (int(img.size[0]/scale), int(img.size[1]/scale)) ) @@ -1608,7 +1753,7 @@ img = Image.open(self.image_path_JPEG).convert('L') width, height = img.size except IOError: - pywikibot.warning(u'unknown file type [_recognize_OpticalCodes_dmtxNzbar]') + pywikibot.warning(u'unknown file type [_recognize_OpticalCodes]') return scanner = zbar.ImageScanner() @@ -1636,7 +1781,7 @@ self._features['OpticalCodes'] = result return - def _detect_Chessboard_CV(self): + def _detect_Chessboard(self): # Chessboard (opencv reference detector) # http://www.c-plusplus.de/forum/273920-full # http://www.youtube.com/watch?v=bV-jAnQ-tvw @@ -1656,10 +1801,10 @@ #scale = max([1., np.average(np.array(im.shape)[0:2]/500.)]) #scale = max([1., np.average(np.array(im.shape)[0:2]/450.)]) except IOError: - pywikibot.warning(u'unknown file type [_detect_Chessboard_CV]') + pywikibot.warning(u'unknown file type [_detect_Chessboard]') return except AttributeError: - pywikibot.warning(u'unknown file type [_detect_Chessboard_CV]') + pywikibot.warning(u'unknown file type [_detect_Chessboard]') return smallImg = np.empty( (cv.Round(im.shape[1]/scale), cv.Round(im.shape[0]/scale)), dtype=np.uint8 ) @@ -1875,7 +2020,7 @@ coords2D = np.dot((cm), coords) perp = coords - origin if hacky: - # for '_detect_Chessboard_CV' but looks a bit strange ... may be wrong?! + # for '_detect_Chessboard' but looks a bit strange ... may be wrong?! mat = coords2D - origin2D mat = mat/max([np.linalg.norm(mat[:,i]) for i in range(3)]) else: @@ -2138,7 +2283,7 @@ self._features['Faces'] += data return - def _detect_History_EXIF(self): + def _detect_History(self): res = self._util_get_DataTags_EXIF() #a = [] @@ -2224,7 +2369,7 @@ drop.append( i1 ) elif (ar2 >= thsr) and (i1 not in drop): drop.append( i2 ) - # from '_detect_Faces_CV()' + # from '_detect_Faces()' if overlap: if (r2[0] <= c1[0] <= (r2[0] + r2[2])) and \ (r2[1] <= c1[1] <= (r2[1] + r2[3])) and (i2 not in drop): @@ -2243,17 +2388,17 @@ return (regs, drop) -class PngFile(JpegFile): +class _PngFile(_JpegFile): pass -class GifFile(JpegFile): +class _GifFile(_JpegFile): pass -class TiffFile(JpegFile): +class _TiffFile(_JpegFile): pass -class XcfFile(JpegFile): +class _XcfFile(_JpegFile): def _convert(self): # Very few programs other than GIMP read XCF files. This is by design # from the GIMP developers, the format is not really documented or @@ -2282,19 +2427,19 @@ as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - result = { 'Format': u'%s' % self.image_mime[1].upper(), + result = { 'Format': u'%s' % self.file_mime[1].upper(), # DO NOT use ImageMagick (identify) instead of PIL to get these info !! 'Pages': 0, 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), } #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) return -class SvgFile(JpegFile): +class _SvgFile(_JpegFile): def _convert(self): # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) # (Mediawiki uses librsvg too: http://commons.wikimedia.org/wiki/SVG#SVGs_in_MediaWiki) @@ -2358,19 +2503,19 @@ # may be set {{validSVG}} also or do something in bot template to # recognize 'Format=SVG (valid)' ... 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), }) + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), }) #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) return -class PdfFile(JpegFile): +class _PdfFile(_JpegFile): def getFeatures(self): # optical and other text recognition (tesseract & ocropus, ...) - self._detect_EmbeddedText_poppler() -# self._recognize_OpticalText_ocropus() + self._detect_EmbeddedText() +# self._recognize_OpticalText() # (may be just classify as 'contains text', may be store text, e.g. to wikisource) return self._features @@ -2383,7 +2528,7 @@ # http://vermeulen.ca/python-pdf.html # http://code.activestate.com/recipes/511465-pure-python-pdf-to-text-converte… # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… - if self.image_fileext == u'.pdf': + if os.path.splitext(self.image_filename)[1].lower() == u'.pdf': pass # MIME: 'application/pdf; charset=binary' @@ -2402,8 +2547,8 @@ 'Palette': u'-', 'Pages': pc, 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), } #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) @@ -2411,7 +2556,7 @@ # ./run-test (ocropus/ocropy) # (in fact all scripts/executables used here are pure python scripts!!!) - def _recognize_OpticalText_ocropus(self): + def _recognize_OpticalText(self): # optical text recognition (tesseract & ocropus, ...) # (no full recognition but - at least - just classify as 'contains text') # http://www.claraocr.org/de/ocr/ocr-software/open-source-ocr.html @@ -2475,7 +2620,7 @@ #print data pywikibot.output(data) - def _detect_EmbeddedText_poppler(self): + def _detect_EmbeddedText(self): # may be also: http://www.reportlab.com/software/opensource/rl-toolkit/ # poppler pdftotext/pdfimages @@ -2538,10 +2683,10 @@ # pdfinterp.process_pdf(rsrcmgr, device, fp, set(), maxpages=0, password='', # caching=True, check_extractable=False) #except AssertionError: - # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') + # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText]') # return #except TypeError: - # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') + # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText]') # return #fp.close() #device.close() @@ -2561,17 +2706,17 @@ return -#class DjvuFile(JpegFile): +#class DjvuFile(_JpegFile): # pass -class OggFile(JpegFile): +class _OggFile(_JpegFile): def getFeatures(self): # general handling of all audio and video formats - self._detect_Streams_FFMPEG() + self._detect_Streams() # general audio feature extraction -# self._detect_AudioFeatures_YAAFE() +# self._detect_AudioFeatures() return self._features @@ -2588,14 +2733,14 @@ result = { 'Format': u'%s' % d['format']['format_name'].upper(), 'Pages': 0, 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), } #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) return - def _detect_Streams_FFMPEG(self): + def _detect_Streams(self): # audio and video streams files (ogv, oga, ...) d = self._util_get_DataStreams_FFMPEG() if not d: @@ -2661,7 +2806,7 @@ return self._buffer_FFMPEG - def _detect_AudioFeatures_YAAFE(self): + def _detect_AudioFeatures(self): # http://yaafe.sourceforge.net/manual/tools.html # http://yaafe.sourceforge.net/manual/quickstart.html - yaafe.py # ( help: yaafe.py -h / features: yaafe.py -l ) @@ -2813,9 +2958,9 @@ return -class MidiFile(UnknownFile): +class _MidiFile(_UnknownFile): def getFeatures(self): - self._detect_AudioFeatures_MUSIC21() # Audio + self._detect_AudioFeatures() # Audio return self._features def _detect_HeaderAndMetadata(self): @@ -2824,7 +2969,7 @@ # extract data from midi file # http://valentin.dasdeck.com/midi/midifile.htm # http://stackoverflow.com/questions/3943149/reading-and-interpreting-data-fr… - ba = bytearray(open(self.filename, 'rb').read()) + ba = bytearray(open(self.file_name, 'rb').read()) i = -1 for key, data in [('Text', '\x01'), ('Copyright', '\x02'), ('Lyrics', '\x05')]: result[key] = [] @@ -2853,7 +2998,7 @@ import _music21 as music21 try: - s = music21.converter.parse(self.filename) + s = music21.converter.parse(self.file_name) if s.metadata: pywikibot.output(unicode(s.metadata)) result.update(s.metadata) @@ -2869,27 +3014,27 @@ as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - result = { 'Format': u'%s' % self.image_mime[1].upper(), + result = { 'Format': u'%s' % self.file_mime[1].upper(), 'Pages': 0, 'Dimensions': self.image_size, - 'Filesize': os.path.getsize(self.filename), - 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } + 'Filesize': os.path.getsize(self.file_name), + 'MIME': u'%s/%s' % tuple(self.file_mime[:2]), } #self._properties['Properties'] = [result] self._properties['Properties'][0].update(result) return # midi audio feature extraction - def _detect_AudioFeatures_MUSIC21(self): + def _detect_AudioFeatures(self): import _music21 as music21 #music21.features.jSymbolic.getCompletionStats() try: #audiofile = '/home/ursin/Desktop/3_Ships.mid' - #s = music21.midi.translate.midiFilePathToStream(self.filename) - s = music21.converter.parse(self.filename) + #s = music21.midi.translate.midiFilePathToStream(self.file_name) + s = music21.converter.parse(self.file_name) except music21.midi.base.MidiException: - pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') + pywikibot.warning(u'unknown file type [_detect_AudioFeatures]') return #fs = music21.features.jSymbolic.extractorsById @@ -2931,21 +3076,34 @@ return -FILETYPES = { '*': UnknownFile, - ( 'image', 'jpeg'): JpegFile, - ( 'image', 'png'): PngFile, - ( 'image', 'gif'): GifFile, - ( 'image', 'tiff'): TiffFile, - ( 'image', 'x-xcf'): XcfFile, - ( 'image', 'svg+xml'): SvgFile, - ('application', 'pdf'): PdfFile, +_FILETYPES = { '*': _UnknownFile, + ( 'image', 'jpeg'): _JpegFile, + ( 'image', 'png'): _PngFile, + ( 'image', 'gif'): _GifFile, + ( 'image', 'tiff'): _TiffFile, + ( 'image', 'x-xcf'): _XcfFile, + ( 'image', 'svg+xml'): _SvgFile, + ('application', 'pdf'): _PdfFile, # djvu: python-djvulibre or python-djvu for djvu support # http://pypi.python.org/pypi/python-djvulibre/0.3.9 -# ( 'image', 'vnd.djvu'): DjvuFile, - ('application', 'ogg'): OggFile, - ( 'audio', 'midi'): MidiFile,} +# ( 'image', 'vnd.djvu'): DjvuFile, + ('application', 'ogg'): _OggFile, + ( 'audio', 'midi'): _MidiFile,} +def GenericFile(file_name): + # 'magic' (libmagic) + m = magic.open(magic.MAGIC_MIME) # or 'magic.MAGIC_NONE' + m.load() + file_mime = re.split('[/;\s]', m.file(file_name)) + mime = mimetypes.guess_all_extensions('%s/%s' % tuple(file_mime[0:2])) + if mime and (os.path.splitext(file_name)[1].lower() not in mime): + pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) + # split detection and extraction according to file types; _JpegFile, ... + GenericFile = _FILETYPES.get(tuple(file_mime[:2]), _FILETYPES['*']) + return GenericFile(file_name, file_mime) + + # all classification and categorization methods and definitions - default variation # use simplest classification I can think of (self-made) and do categorization # mostly based on filtered/reported features @@ -2957,7 +3115,7 @@ #_thrshld_guesses = 0.1 _thrshld_default = 0.75 - # for '_detect_Trained_CV' + # for '_detect_Trained' cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), (u'Torsos', 'haarcascade_upperbody.xml'), (u'Ears', 'haarcascade_mcs_leftear.xml'), @@ -3387,7 +3545,6 @@ pywikibot.output(u'Processing media %s ...' % self.image.title(asLink=True)) image_filename = os.path.split(self.image.fileUrl())[-1] - image_fileext = os.path.splitext(image_filename)[1] self.image_path = urllib2.quote(os.path.join(scriptdir, ('cache/' + image_filename[-128:]))) self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) @@ -3411,15 +3568,6 @@ f.write( data ) f.close() - # 'magic' (libmagic) - m = magic.open(magic.MAGIC_MIME) # or 'magic.MAGIC_NONE' - m.load() - self.image_mime = re.split('[/;\s]', m.file(self.image_path)) - #self.image_size = (None, None) - mime = mimetypes.guess_all_extensions('%s/%s' % tuple(self.image_mime[0:2])) - if mime and (image_fileext.lower() not in mime): - pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) - # LOOK ALSO AT: checkimages.CatImagesBot.checkStep # (and category scripts/bots too...) def checkStep(self): @@ -3673,7 +3821,7 @@ return u" | %s = %s" % (key, self._output_format(value)) def _make_markerblock(self, res, size, structure=['Position'], line='solid'): - # same as in '_detect_Faces_CV' + # same as in '_detect_Faces' colors = [ (0,0,255), (0,128,255), (0,255,255), @@ -3739,17 +3887,13 @@ # gather data from all information interfaces def gatherFeatures(self): - # split detection and extraction according to file types; JpegFile, ... - TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) - with TypeFile(self.image_path) as tf: - tf.image_mime = self.image_mime - tf.image = self.image + # split detection and extraction according to file types; _JpegFile, ... + with GenericFile(self.image_path) as gf: + gf.image = self.image # patch for _SvgFile needing url for func in ['getProperties', 'getFeatures']: - result = getattr(tf, func)() + result = getattr(gf, func)() self._info.update(result) - print self._info - #print tf.__dict__ - self.image_size = tf.image_size + self.image_size = gf.image_size def _existInformation(self, info, ignore = ['Properties', 'Metadata', 'ColorAverage']): result = [] @@ -4185,7 +4329,7 @@ linear_svm = mlpy.LibSvm(kernel_type='linear') # new linear SVM instance linear_svm.learn(z, y) # learn from principal components - # !!! train also BoW (bag-of-words) in '_detectclassify_ObjectAll_CV' resp. 'opencv.BoWclassify.main' !!! + # !!! train also BoW (bag-of-words) in '_detectclassify_ObjectAll' resp. 'opencv.BoWclassify.main' !!! xmin, xmax = z[:,0].min()-0.1, z[:,0].max()+0.1 ymin, ymax = z[:,1].min()-0.1, z[:,1].max()+0.1

1 0

SVN: [11623] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 08 Jun '13

08 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11623 Revision: 11623 Author: drtrigon Date: 2013-06-07 22:29:18 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; code cleanup and class conversion more or less finished (method renaming not done yet) new feature; initial audio midi support (music21) and own metadata part (Category:MIDI files created with GNU LilyPond) Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-07 18:21:31 UTC (rev 11622) +++ trunk/pywikipedia/catimages.py 2013-06-07 22:29:18 UTC (rev 11623) @@ -149,7 +149,27 @@ self.filename = filename self.image_size = (None, None) - self._info = {} + # available file properties and metadata + self._properties = { 'Properties': [{'Format': u'-', 'Pages': 0}], + 'Metadata': [], } + # available feature to extract + self._features = { 'ColorAverage': [], + 'ColorRegions': [], + 'Faces': [], + 'People': [], + 'OpticalCodes': [], + 'Chessboard': [], + 'History': [], + 'Text': [], + 'Streams': [], + 'Audio': [], + 'Legs': [], + 'Hands': [], + 'Torsos': [], + 'Ears': [], + 'Eyes': [], + 'Automobiles': [], + 'Classify': [], } def __enter__(self): return self @@ -158,22 +178,22 @@ pass def getProperties(self): - result = {} - result.update( self._detect_HeaderAndMetadata() ) # Metadata - result.update( self._detect_Properties_PIL() ) # Properties - return result + self._detect_HeaderAndMetadata() # Metadata + self._detect_Properties() # Properties + return self._properties def getFeatures(self): pywikibot.warning(u"File format '%s/%s' not supported (yet)!" % tuple(self.image_mime[:2])) + return self._features def _detect_HeaderAndMetadata(self): # check/look into the file by midnight commander (mc) # https://pypi.python.org/pypi/hachoir-metadata - return {} + pass - def _detect_Properties_PIL(self): + def _detect_Properties(self): # get mime-type file-size, ... - return {} + pass class JpegFile(UnknownFile): @@ -197,14 +217,14 @@ self.image_filename = os.path.split(self.filename)[-1] self.image_fileext = os.path.splitext(self.image_filename)[1] - self.image_path = os.path.join(scriptdir, ('cache/' + self.image_filename[-128:])) + self.image_path = self.filename self.image_path_JPEG = self.image_path + '.jpg' self._convert() def __exit__(self, type, value, traceback): - if os.path.exists(self.image_path): - os.remove( self.image_path ) + #if os.path.exists(self.image_path): + # os.remove( self.image_path ) if os.path.exists(self.image_path_JPEG): os.remove( self.image_path_JPEG ) #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") @@ -220,9 +240,9 @@ # Face via Landmark(s) # self._detect_FaceLandmark_xBOB() # exclude duplicates (CV and EXIF) - faces = [item['Position'] for item in self._info['Faces']] + faces = [item['Position'] for item in self._features['Faces']] for i in self._util_merge_Regions(faces)[1]: - del self._info['Faces'][i] + del self._features['Faces'][i] # Segments and colors self._detect_SegmentColors_JSEGnPIL() @@ -241,11 +261,6 @@ for cf in self.cascade_files: self._detect_Trained_CV(*cf) - # optical and other text recognition (tesseract & ocropus, ...) - self._detect_EmbeddedText_poppler() -# self._recognize_OpticalText_ocropus() - # (may be just classify as 'contains text', may be store text, e.g. to wikisource) - # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) self._recognize_OpticalCodes_dmtxNzbar() @@ -260,6 +275,8 @@ # general file EXIF history information self._detect_History_EXIF() + + return self._features # supports a lot of different file types thanks to PIL def _convert(self): @@ -293,18 +310,18 @@ # http://code.google.com/p/pylibtiff/ # MIME: 'image/jpeg; charset=binary', ... - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} try: i = Image.open(self.image_path) except IOError: pywikibot.warning(u'unknown file type [JpegFile]') - return {'Properties': [result]} + return # http://mail.python.org/pipermail/image-sig/1999-May/000740.html pc=0 # count number of pages @@ -325,23 +342,22 @@ #self.image_size = i.size - result = { #'bands': i.getbands(), - #'bbox': i.getbbox(), - 'Format': i.format, - 'Mode': i.mode, - #'info': i.info, - #'stat': os.stat(self.image_path), - 'Palette': str(len(i.palette.palette)) if i.palette else u'-', - 'Pages': pc, } + result.update({ #'bands': i.getbands(), + #'bbox': i.getbbox(), + 'Format': i.format, + 'Mode': i.mode, + #'info': i.info, + #'stat': os.stat(self.image_path), + 'Palette': str(len(i.palette.palette)) if i.palette else u'-', + 'Pages': pc, + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), }) - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html def _detect_Faces_CV(self): @@ -354,11 +370,6 @@ # http://blog.jozilla.net/2008/06/27/fun-with-python-opencv-and-face-detectio… # http://www.cognotics.com/opencv/servo_2007_series/part_4/index.html - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - # https://code.ros.org/trac/opencv/browser/trunk/opencv_extra/testdata/gpu/ha… xml = os.path.join(scriptdir, 'externals/opencv/haarcascades/haarcascade_eye_tree_eyeglasses.xml') #xml = os.path.join(scriptdir, 'externals/opencv/haarcascades/haarcascade_eye.xml') @@ -402,7 +413,6 @@ raise IOError(u"No such file: '%s'" % xml) cascaderightear = cv2.CascadeClassifier(xml) - #self._info['Faces'] = [] scale = 1. # So, to find an object of an unknown size in the image the scan # procedure should be done several times at different scales. @@ -651,7 +661,7 @@ # cv2.imwrite( image_path_new, img ) #return faces.tolist() - self._info['Faces'] += result + self._features['Faces'] += result return def _util_get_Pose_solvePnP(self, D3points, D2points, shape): @@ -728,7 +738,6 @@ """Prints the locations of any face landmark(s) found, respective converts them to usual face position data""" - #self._info['Faces'] = [] scale = 1. try: #video = bob.io.VideoReader(self.image_path_JPEG.encode('utf-8')) @@ -794,7 +803,7 @@ #cv2.imshow("people detector", img) #cv2.waitKey() - self._info['Faces'] += result + self._features['Faces'] += result return # .../opencv/samples/cpp/peopledetect.cpp @@ -806,7 +815,6 @@ # http://opencv.willowgarage.com/documentation/cpp/basic_structures.html # http://www.pygtk.org/docs/pygtk/class-gdkrectangle.html - self._info['People'] = [] scale = 1. try: img = cv2.imread(self.image_path_JPEG, cv.CV_LOAD_IMAGE_COLOR) @@ -891,22 +899,15 @@ #cv2.imshow("people detector", img) #c = cv2.waitKey(0) & 255 - self._info['People'] = result + self._features['People'] = result return def _detect_Geometry_CV(self): - self._info['Geometry'] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - result = self._util_get_Geometry_CVnSCIPY() - self._info['Geometry'] = [{'Lines': result['Lines'], - 'Circles': result['Circles'], - 'Corners': result['Corners'],}] + self._features['Geometry'] = [{'Lines': result['Lines'], + 'Circles': result['Circles'], + 'Corners': result['Corners'],}] return # https://code.ros.org/trac/opencv/browser/trunk/opencv/samples/python/houghl… @@ -1083,8 +1084,6 @@ # parts of code here should/have to be placed into e.g. a own # class in 'dtbext/opencv/__init__.py' script/module - self._info['Classify'] = [] - trained = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', @@ -1123,7 +1122,7 @@ # http://www.xrce.xerox.com/layout/set/print/content/download/18763/134049/fi… # http://people.csail.mit.edu/torralba/shortCourseRLOC/index.html - self._info['Classify'] = [dict([ (trained[i], r) for i, r in enumerate(result) ])] + self._features['Classify'] = [dict([ (trained[i], r) for i, r in enumerate(result) ])] return def _detectclassify_ObjectAll_PYWT(self): @@ -1167,13 +1166,6 @@ # http://code.google.com/p/pymeanshift/wiki/Examples # (http://pythonvision.org/basic-tutorial, http://luispedro.org/software/mahotas, http://packages.python.org/pymorph/) def _detect_SegmentColors_JSEGnPIL(self): # may be SLIC other other too... - self._info['ColorRegions'] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - try: #im = Image.open(self.image_path).convert(mode = 'RGB') im = Image.open(self.image_path_JPEG) @@ -1220,7 +1212,7 @@ result.append( data ) i += 1 - self._info['ColorRegions'] = result + self._features['ColorRegions'] = result return # http://stackoverflow.com/questions/2270874/image-color-detection-using-pyth… @@ -1230,13 +1222,6 @@ # http://en.wikipedia.org/wiki/Color_difference # http://www.farb-tabelle.de/en/table-of-color.htm def _detect_AverageColor_PILnCV(self): - self._info['ColorAverage'] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - try: # we need to have 3 channels (but e.g. grayscale 'P' has only 1) #i = Image.open(self.image_path).convert(mode = 'RGB') @@ -1249,7 +1234,7 @@ result = self._util_average_Color_colormath(h) result['Gradient'] = self._util_get_Geometry_CVnSCIPY().get('Edge_Ratio', None) or '-' result['FFT_Peaks'] = self._util_get_Geometry_CVnSCIPY().get('FFT_Peaks', None) or '-' - self._info['ColorAverage'] = [result] + self._features['ColorAverage'] = [result] return # http://stackoverflow.com/questions/2270874/image-color-detection-using-pyth… @@ -1509,13 +1494,6 @@ # analogue to face detection: - self._info[info_desc] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - # http://tutorial-haartraining.googlecode.com/svn/trunk/data/haarcascades/ # or own xml files trained onto specific file database/set xml = os.path.join(scriptdir, ('externals/opencv/haarcascades/' + cascade_file)) @@ -1560,170 +1538,9 @@ # generic detection ... - self._info[info_desc] = result + self._features[info_desc] = result return - # ./run-test (ocropus/ocropy) - # (in fact all scripts/executables used here are pure python scripts!!!) - def _recognize_OpticalText_ocropus(self): - # optical text recognition (tesseract & ocropus, ...) - # (no full recognition but - at least - just classify as 'contains text') - # http://www.claraocr.org/de/ocr/ocr-software/open-source-ocr.html - # https://github.com/edsu/ocropy - # http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/Doku#Categorization - # Usage:tesseract imagename outputbase [-l lang] [configfile [[+|-]varfile]...] - # tesseract imagename.tif output - - # (it's simpler to run the scripts/executables in own environment/interpreter...) - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): - return - - path = os.path.join(scriptdir, 'dtbext/_ocropus/ocropy') - - curdir = os.path.abspath(os.curdir) - os.chdir(path) - - # binarization - if os.path.exists(os.path.join(path, "temp")): - shutil.rmtree(os.path.join(path, "temp")) - if os.system("ocropus-nlbin %s -o %s" % (self.image_path_JPEG, os.path.join(path, "temp"))): - raise ImportError("ocropus not found!") - - # page level segmentation - if os.system("ocropus-gpageseg --minscale 6.0 '%s'" % os.path.join(path, "temp/????.bin.png")): - # detection error - return - - # raw text line recognition - if os.system("ocropus-lattices --writebestpath '%s'" % os.path.join(path, "temp/????/??????.bin.png")): - # detection error - return - - # language model application - # (optional - improve the raw results by applying a pretrained model) - os.environ['OCROPUS_DATA'] = os.path.join(path, "models/") - if os.system("ocropus-ngraphs '%s'" % os.path.join(path, "temp/????/??????.lattice")): - # detection error - return - - # create hOCR output - if os.system("ocropus-hocr '%s' -o %s" % (os.path.join(path, "temp/????.bin.png"), os.path.join(path, "temp.html"))): - # detection error - return - - ## 'create HTML for debugging (use "firefox temp/index.html" to view)' - ## (optional - generate human readable debug output) - #if os.system("ocropus-visualize-results %s" % os.path.join(path, "temp")): - # # detection error - # return - - # "to see recognition results, type: firefox temp.html" - # "to see details on the recognition process, type: firefox temp/index.html" - tmpfile = open(os.path.join(path, "temp.html"), 'r') - data = tmpfile.read() - tmpfile.close() - - shutil.rmtree(os.path.join(path, "temp")) - os.remove(os.path.join(path, "temp.html")) - - os.chdir(curdir) - - #print data - pywikibot.output(data) - - def _detect_EmbeddedText_poppler(self): - # may be also: http://www.reportlab.com/software/opensource/rl-toolkit/ - - self._info['Text'] = [] - - if not (self.image_mime[1] == 'pdf'): - return - - # poppler pdftotext/pdfimages - # (similar as in '_util_get_DataTags_EXIF' but with stderr and no json output) - # http://poppler.freedesktop.org/ - # http://www.izzycode.com/bash/how-to-install-pdf2text-on-centos-fedora-redha… - # MIGHT BE BETTER TO USE AS PYTHON MODULE: - # https://launchpad.net/poppler-python/ - # http://stackoverflow.com/questions/2732178/extracting-text-from-pdf-with-po… - # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… - #proc = Popen("pdftotext -layout %s %s" % (self.image_path, self.image_path+'.txt'), - proc = Popen("pdftotext %s %s" % (self.image_path, self.image_path+'.txt'), - shell=True, stderr=PIPE)#.stderr.readlines() - proc.wait() - if proc.returncode: - raise ImportError("pdftotext not found!") - data = open(self.image_path+'.txt', 'r').readlines() - os.remove( self.image_path+'.txt' ) - -# self._content_text = data - (s1, l1) = (len(u''.join(data)), len(data)) - - tmp_path = os.path.join(os.environ.get('TMP', '/tmp'), 'DrTrigonBot/') - os.mkdir( tmp_path ) -# switch this part off since 'pdfimages' (on toolserver) is too old; TS-1449 -# proc = Popen("pdfimages -p %s %s/" % (self.image_path, tmp_path), - proc = Popen("pdfimages %s %s/" % (self.image_path, tmp_path), - shell=True, stderr=PIPE)#.stderr.readlines() - proc.wait() - if proc.returncode: - raise ImportError("pdfimages not found!") - images = os.listdir( tmp_path ) -# pages = set() - for f in images: -# pages.add( int(f.split('-')[1]) ) - os.remove( os.path.join(tmp_path, f) ) - os.rmdir( tmp_path ) - - ## pdfminer (tools/pdf2txt.py) - ## http://denis.papathanasiou.org/?p=343 (for layout and images) - #debug = 0 - #laparams = layout.LAParams() - ## - #pdfparser.PDFDocument.debug = debug - #pdfparser.PDFParser.debug = debug - #cmapdb.CMapDB.debug = debug - #pdfinterp.PDFResourceManager.debug = debug - #pdfinterp.PDFPageInterpreter.debug = debug - #pdfdevice.PDFDevice.debug = debug - ## - #rsrcmgr = pdfinterp.PDFResourceManager(caching=True) - #outfp = StringIO.StringIO() - #device = converter.TextConverter(rsrcmgr, outfp, codec='utf-8', laparams=laparams) - ##device = converter.XMLConverter(rsrcmgr, outfp, codec='utf-8', laparams=laparams, outdir=None) - ##device = converter.HTMLConverter(rsrcmgr, outfp, codec='utf-8', scale=1, - ## layoutmode='normal', laparams=laparams, outdir=None) - ##device = pdfdevice.TagExtractor(rsrcmgr, outfp, codec='utf-8') - #fp = file(self.image_path, 'rb') - #try: - # pdfinterp.process_pdf(rsrcmgr, device, fp, set(), maxpages=0, password='', - # caching=True, check_extractable=False) - #except AssertionError: - # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') - # return - #except TypeError: - # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') - # return - #fp.close() - #device.close() - #data = outfp.getvalue().splitlines(True) - # - #(s2, l2) = (len(u''.join(data)), len(data)) - - result = { 'Size': s1, - 'Lines': l1, - #'Data': data, - #'Position': pos, -# 'Images': u'%s (on %s page(s))' % (len(images), len(list(pages))), # pages containing images - 'Images': u'%s' % len(images), - 'Type': u'-', } # 'Type' could be u'OCR' above... - - self._info['Text'] = [result] - - return - def _recognize_OpticalCodes_dmtxNzbar(self): # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) # http://libdmtx.wikidot.com/libdmtx-python-wrapper @@ -1731,13 +1548,6 @@ # http://zbar.sourceforge.net/ # http://pypi.python.org/pypi/zbar - self._info['OpticalCodes'] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - # DataMatrix from pydmtx import DataMatrix # linux distro package (fedora) / TS (debian) @@ -1786,7 +1596,7 @@ 'Type': u'DataMatrix', 'Quality': 10, }) - self._info['OpticalCodes'] = result + self._features['OpticalCodes'] = result # supports many popular symbologies try: @@ -1823,7 +1633,7 @@ # further detection ? - self._info['OpticalCodes'] = result + self._features['OpticalCodes'] = result return def _detect_Chessboard_CV(self): @@ -1832,13 +1642,6 @@ # http://www.youtube.com/watch?v=bV-jAnQ-tvw # http://nullege.com/codes/show/src%40o%40p%40opencvpython-HEAD%40samples%40c… - self._info['Chessboard'] = [] - - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ - (self.image_mime[0] in ['audio']): - return - scale = 1. try: #im = cv.LoadImage(self.image_path_JPEG, cv.CV_LOAD_IMAGE_COLOR) @@ -1877,9 +1680,10 @@ ##cv2.imshow("win", im) ##cv2.waitKey() + result = {} if corners is not None: - self._info['Chessboard'] = [{ 'Corners': [tuple(item[0]) - for item in corners], }] + result = { 'Corners': [tuple(item[0]) for item in corners], } + self._features['Chessboard'] = [result] # TODO: improve chessboard detection (make it more tolerant) # ## http://stackoverflow.com/questions/7624765/converting-an-opencv-image-to-bl… @@ -2019,10 +1823,10 @@ pywikibot.output(u'result for calibrated camera:\n rot=%s\n perp=%s\n perp2D=%s' % (rot.transpose()[0], perp[:,2], ortho)) pywikibot.output(u'nice would be to do the same for uncalibrated/default cam settings') - self._info['Chessboard'][0].update({ - 'Rotation': tuple(rot.transpose()[0]), - 'Perp_Dir' : tuple(perp[:,2]), - 'Perp_Dir_2D': tuple(ortho), }) + result.update({ 'Rotation': tuple(rot.transpose()[0]), + 'Perp_Dir' : tuple(perp[:,2]), + 'Perp_Dir_2D': tuple(ortho), }) + self._features['Chessboard'] = [result] #cv2.imshow("win", im) #cv2.waitKey() @@ -2331,12 +2135,10 @@ # (exclusion of duplicates is done later by '_util_merge_Regions') - self._info['Faces'] += data + self._features['Faces'] += data return def _detect_History_EXIF(self): - self._info['History'] = [] - res = self._util_get_DataTags_EXIF() #a = [] @@ -2372,42 +2174,9 @@ pass i += 1 - self._info['History'] = result + self._features['History'] = result return - def _util_get_DataStreams_FFMPEG(self): - if hasattr(self, '_buffer_FFMPEG'): - return self._buffer_FFMPEG - - # (similar as in '_util_get_DataTags_EXIF') -# switch this part off since 'ffprobe' (on toolserver) is too old; TS-1449 -# data = Popen("ffprobe -v quiet -print_format json -show_format -show_streams %s" % self.image_path, - proc = Popen("ffprobe -v quiet -show_format -show_streams %s" % self.image_path,#.replace('%', '%%'), - shell=True, stdout=PIPE)#.stdout.read() - proc.wait() - if proc.returncode == 127: - raise ImportError("ffprobe (ffmpeg) not found!") - data = proc.stdout.read().strip() -# self._buffer_FFMPEG = json.loads(data) - res, key, cur = {}, '', {} - for item in data.splitlines(): - if (item[0] == '['): - if not (item[1] == '/'): - key = item[1:-1] - cur = {} - if key not in res: - res[key] = [] - else: - res[key].append( cur ) - else: - val = item.split('=') - cur[val[0].strip()] = val[1].strip() - if res: - res = { 'streams': res['STREAM'], 'format': res['FORMAT'][0] } - self._buffer_FFMPEG = res - - return self._buffer_FFMPEG - def _util_merge_Regions(self, regs, sub=False, overlap=False, close=False): # sub=False, overlap=False, close=False ; level 0 ; similar regions, similar position (default) # sub=True, overlap=False, close=False ; level 1 ; region contained in other, any shape/size @@ -2508,25 +2277,23 @@ self.image_size = Image.open(self.image_path_JPEG).size # MIME: 'image/x-xcf; charset=binary' - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} - result = { 'Format': u'%s' % self.image_mime[1].upper() } + result = { 'Format': u'%s' % self.image_mime[1].upper(), # DO NOT use ImageMagick (identify) instead of PIL to get these info !! + 'Pages': 0, + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - class SvgFile(JpegFile): def _convert(self): # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) @@ -2556,11 +2323,11 @@ self.image_path_JPEG = self.image_path # MIME: 'application/xml; charset=utf-8' - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} # similar to PDF page count OR use BeautifulSoup @@ -2584,23 +2351,30 @@ #self.image_size = (svg.props.width, svg.props.height) - result = { 'Format': valid, - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } + result.update({ 'Format': valid, + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, # may be set {{validSVG}} also or do something in bot template to # recognize 'Format=SVG (valid)' ... + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), }) - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - class PdfFile(JpegFile): + def getFeatures(self): + # optical and other text recognition (tesseract & ocropus, ...) + self._detect_EmbeddedText_poppler() +# self._recognize_OpticalText_ocropus() + # (may be just classify as 'contains text', may be store text, e.g. to wikisource) + + return self._features + def _convert(self): # self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) @@ -2613,32 +2387,180 @@ pass # MIME: 'application/pdf; charset=binary' - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} # http://code.activestate.com/recipes/496837-count-pdf-pages/ #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) - result = { 'Format': u'PDF', - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } + result = { 'Format': u'PDF', + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} + # ./run-test (ocropus/ocropy) + # (in fact all scripts/executables used here are pure python scripts!!!) + def _recognize_OpticalText_ocropus(self): + # optical text recognition (tesseract & ocropus, ...) + # (no full recognition but - at least - just classify as 'contains text') + # http://www.claraocr.org/de/ocr/ocr-software/open-source-ocr.html + # https://github.com/edsu/ocropy + # http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/Doku#Categorization + # Usage:tesseract imagename outputbase [-l lang] [configfile [[+|-]varfile]...] + # tesseract imagename.tif output + # (it's simpler to run the scripts/executables in own environment/interpreter...) + path = os.path.join(scriptdir, 'dtbext/_ocropus/ocropy') + + curdir = os.path.abspath(os.curdir) + os.chdir(path) + + # binarization + if os.path.exists(os.path.join(path, "temp")): + shutil.rmtree(os.path.join(path, "temp")) + if os.system("ocropus-nlbin %s -o %s" % (self.image_path_JPEG, os.path.join(path, "temp"))): + raise ImportError("ocropus not found!") + + # page level segmentation + if os.system("ocropus-gpageseg --minscale 6.0 '%s'" % os.path.join(path, "temp/????.bin.png")): + # detection error + return + + # raw text line recognition + if os.system("ocropus-lattices --writebestpath '%s'" % os.path.join(path, "temp/????/??????.bin.png")): + # detection error + return + + # language model application + # (optional - improve the raw results by applying a pretrained model) + os.environ['OCROPUS_DATA'] = os.path.join(path, "models/") + if os.system("ocropus-ngraphs '%s'" % os.path.join(path, "temp/????/??????.lattice")): + # detection error + return + + # create hOCR output + if os.system("ocropus-hocr '%s' -o %s" % (os.path.join(path, "temp/????.bin.png"), os.path.join(path, "temp.html"))): + # detection error + return + + ## 'create HTML for debugging (use "firefox temp/index.html" to view)' + ## (optional - generate human readable debug output) + #if os.system("ocropus-visualize-results %s" % os.path.join(path, "temp")): + # # detection error + # return + + # "to see recognition results, type: firefox temp.html" + # "to see details on the recognition process, type: firefox temp/index.html" + tmpfile = open(os.path.join(path, "temp.html"), 'r') + data = tmpfile.read() + tmpfile.close() + + shutil.rmtree(os.path.join(path, "temp")) + os.remove(os.path.join(path, "temp.html")) + + os.chdir(curdir) + + #print data + pywikibot.output(data) + + def _detect_EmbeddedText_poppler(self): + # may be also: http://www.reportlab.com/software/opensource/rl-toolkit/ + + # poppler pdftotext/pdfimages + # (similar as in '_util_get_DataTags_EXIF' but with stderr and no json output) + # http://poppler.freedesktop.org/ + # http://www.izzycode.com/bash/how-to-install-pdf2text-on-centos-fedora-redha… + # MIGHT BE BETTER TO USE AS PYTHON MODULE: + # https://launchpad.net/poppler-python/ + # http://stackoverflow.com/questions/2732178/extracting-text-from-pdf-with-po… + # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… + #proc = Popen("pdftotext -layout %s %s" % (self.image_path, self.image_path+'.txt'), + proc = Popen("pdftotext %s %s" % (self.image_path, self.image_path+'.txt'), + shell=True, stderr=PIPE)#.stderr.readlines() + proc.wait() + if proc.returncode: + raise ImportError("pdftotext not found!") + data = open(self.image_path+'.txt', 'r').readlines() + os.remove( self.image_path+'.txt' ) + +# self._content_text = data + (s1, l1) = (len(u''.join(data)), len(data)) + + tmp_path = os.path.join(os.environ.get('TMP', '/tmp'), 'DrTrigonBot/') + os.mkdir( tmp_path ) +# switch this part off since 'pdfimages' (on toolserver) is too old; TS-1449 +# proc = Popen("pdfimages -p %s %s/" % (self.image_path, tmp_path), + proc = Popen("pdfimages %s %s/" % (self.image_path, tmp_path), + shell=True, stderr=PIPE)#.stderr.readlines() + proc.wait() + if proc.returncode: + raise ImportError("pdfimages not found!") + images = os.listdir( tmp_path ) +# pages = set() + for f in images: +# pages.add( int(f.split('-')[1]) ) + os.remove( os.path.join(tmp_path, f) ) + os.rmdir( tmp_path ) + + ## pdfminer (tools/pdf2txt.py) + ## http://denis.papathanasiou.org/?p=343 (for layout and images) + #debug = 0 + #laparams = layout.LAParams() + ## + #pdfparser.PDFDocument.debug = debug + #pdfparser.PDFParser.debug = debug + #cmapdb.CMapDB.debug = debug + #pdfinterp.PDFResourceManager.debug = debug + #pdfinterp.PDFPageInterpreter.debug = debug + #pdfdevice.PDFDevice.debug = debug + ## + #rsrcmgr = pdfinterp.PDFResourceManager(caching=True) + #outfp = StringIO.StringIO() + #device = converter.TextConverter(rsrcmgr, outfp, codec='utf-8', laparams=laparams) + ##device = converter.XMLConverter(rsrcmgr, outfp, codec='utf-8', laparams=laparams, outdir=None) + ##device = converter.HTMLConverter(rsrcmgr, outfp, codec='utf-8', scale=1, + ## layoutmode='normal', laparams=laparams, outdir=None) + ##device = pdfdevice.TagExtractor(rsrcmgr, outfp, codec='utf-8') + #fp = file(self.image_path, 'rb') + #try: + # pdfinterp.process_pdf(rsrcmgr, device, fp, set(), maxpages=0, password='', + # caching=True, check_extractable=False) + #except AssertionError: + # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') + # return + #except TypeError: + # pywikibot.warning(u'pdfminer missed, may be corrupt [_detect_EmbeddedText_poppler]') + # return + #fp.close() + #device.close() + #data = outfp.getvalue().splitlines(True) + # + #(s2, l2) = (len(u''.join(data)), len(data)) + + result = { 'Size': s1, + 'Lines': l1, + #'Data': data, + #'Position': pos, +# 'Images': u'%s (on %s page(s))' % (len(images), len(list(pages))), # pages containing images + 'Images': u'%s' % len(images), + 'Type': u'-', } # 'Type' could be u'OCR' above... + + self._features['Text'] = [result] + return + + #class DjvuFile(JpegFile): # pass @@ -2651,35 +2573,29 @@ # general audio feature extraction # self._detect_AudioFeatures_YAAFE() + return self._features + # MIME: 'application/ogg; charset=binary' - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) d = self._util_get_DataStreams_FFMPEG() #print d - result['Format'] = u'%s' % d['format']['format_name'].upper() - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + result = { 'Format': u'%s' % d['format']['format_name'].upper(), + 'Pages': 0, + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return def _detect_Streams_FFMPEG(self): - self._info['Streams'] = [] - - # skip file formats that interfere and can cause strange results (pdf is oga?!) - # or file formats not supported (yet?) - if (self.image_mime[1] in ['pdf']) or (self.image_fileext in [u'.svg']): - return - # audio and video streams files (ogv, oga, ...) d = self._util_get_DataStreams_FFMPEG() if not d: @@ -2707,10 +2623,44 @@ 'Dimensions': dim or (None, None), }) - if 'image' not in d["format"]["format_name"]: - self._info['Streams'] = result + if 'image' in d["format"]["format_name"]: + result = [] + self._features['Streams'] = result return + def _util_get_DataStreams_FFMPEG(self): + if hasattr(self, '_buffer_FFMPEG'): + return self._buffer_FFMPEG + + # (similar as in '_util_get_DataTags_EXIF') +# switch this part off since 'ffprobe' (on toolserver) is too old; TS-1449 +# data = Popen("ffprobe -v quiet -print_format json -show_format -show_streams %s" % self.image_path, + proc = Popen("ffprobe -v quiet -show_format -show_streams %s" % self.image_path,#.replace('%', '%%'), + shell=True, stdout=PIPE)#.stdout.read() + proc.wait() + if proc.returncode == 127: + raise ImportError("ffprobe (ffmpeg) not found!") + data = proc.stdout.read().strip() +# self._buffer_FFMPEG = json.loads(data) + res, key, cur = {}, '', {} + for item in data.splitlines(): + if (item[0] == '['): + if not (item[1] == '/'): + key = item[1:-1] + cur = {} + if key not in res: + res[key] = [] + else: + res[key].append( cur ) + else: + val = item.split('=') + cur[val[0].strip()] = val[1].strip() + if res: + res = { 'streams': res['STREAM'], 'format': res['FORMAT'][0] } + self._buffer_FFMPEG = res + + return self._buffer_FFMPEG + def _detect_AudioFeatures_YAAFE(self): # http://yaafe.sourceforge.net/manual/tools.html # http://yaafe.sourceforge.net/manual/quickstart.html - yaafe.py @@ -2763,12 +2713,6 @@ # $ export YAAFE_PATH=/home/ursin/Desktop/yaafe-v0.64/src_python/ # $ export PYTHONPATH=/home/ursin/Desktop/yaafe-v0.64/src_python - # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg']):#, 'midi']): - return - - self._info['Audio'] = [] - import yaafelib as yaafe # use WAV, OGG, MP3 (and others) audio file formats @@ -2865,15 +2809,14 @@ os.remove(fn) # remove folder too... - self._info['Audio'] = [data] + self._features['Audio'] = [data] return class MidiFile(UnknownFile): def getFeatures(self): - result = {} - result.update( self._detect_AudioFeatures_MUSIC21() ) # Audio - return result + self._detect_AudioFeatures_MUSIC21() # Audio + return self._features def _detect_HeaderAndMetadata(self): result = {} @@ -2895,51 +2838,56 @@ result[key].append(ba[i+3:e].decode('latin-1').strip()) result[key] = u'\n'.join(result[key]) + ## find specific info in extracted data + #print [item.strip() for item in re.findall('Generated .*?\n', result['Text'])] + ##u"Cr'eateur: GNU LilyPond 2.0.1" + #import dateutil.parser + #dates = [] + #for line in result['Text'].splitlines(): + # # http://stackoverflow.com/questions/3276180/extracting-date-from-a-string-in… + # try: + # dates.append(dateutil.parser.parse(line, fuzzy=True).isoformat(' ').decode('utf-8')) + # except ValueError: + # pass + #print dates + import _music21 as music21 try: s = music21.converter.parse(self.filename) if s.metadata: pywikibot.output(unicode(s.metadata)) - result['Metadata'].update(s.metadata) + result.update(s.metadata) except music21.midi.base.MidiException: pass - self._info['Metadata'] = [result] - return {'Metadata': [result]} + self._properties['Metadata'] = [result] + return # MIME: 'audio/midi; charset=binary' - def _detect_Properties_PIL(self): + def _detect_Properties(self): """Retrieve as much file property info possible, especially the same as commons does in order to compare if those libraries (ImageMagick, ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} - result['Format'] = u'%s' % self.image_mime[1].upper() + result = { 'Format': u'%s' % self.image_mime[1].upper(), + 'Pages': 0, + 'Dimensions': self.image_size, + 'Filesize': os.path.getsize(self.filename), + 'MIME': u'%s/%s' % tuple(self.image_mime[:2]), } - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + #self._properties['Properties'] = [result] + self._properties['Properties'][0].update(result) + return - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - # midi audio feature extraction def _detect_AudioFeatures_MUSIC21(self): - # skip file formats not supported - if (self.image_mime[1] not in ['midi']): - return - import _music21 as music21 - #audiofile = '/home/ursin/Desktop/3_Ships.mid' - audiofile = self.image_path - #music21.features.jSymbolic.getCompletionStats() try: - #s = music21.midi.translate.midiFilePathToStream(audiofile) - s = music21.converter.parse(audiofile) + #audiofile = '/home/ursin/Desktop/3_Ships.mid' + #s = music21.midi.translate.midiFilePathToStream(self.filename) + s = music21.converter.parse(self.filename) except music21.midi.base.MidiException: pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') return @@ -2979,8 +2927,8 @@ #print s.seconds #print s.secondsMap - self._info['Audio'] += [data] - return {'Audio': [data]} + self._features['Audio'] = [data] + return FILETYPES = { '*': UnknownFile, @@ -3218,21 +3166,14 @@ return (u'Graphics', bool(relevance)) -# # Category:MIDI files created with GNU LilyPond -# def _cat_audio_MIDIfilescreatedwithGNULilyPond(self): -# # find metadata in extracted data -# print [item.strip() for item in re.findall('Generated .*?\n', u'\n'.join(result['Text']))] -# #u"Cr'eateur: GNU LilyPond 2.0.1" -# import dateutil.parser -# dates = [] -# for line in result['Text']: -# # http://stackoverflow.com/questions/3276180/extracting-date-from-a-string-in… -# try: -# dates.append(dateutil.parser.parse(line, fuzzy=True).isoformat(' ').decode('utf-8')) -# except ValueError: -# pass -# print dates + # Category:MIDI files created with GNU LilyPond + def _cat_meta_MIDIfilescreatedwithGNULilyPond(self): + result = self._info_filter['Metadata'] + relevance = (u"Generated automatically by: GNU LilyPond" in + result[0]['Text']) + return (u'MIDI files created with GNU LilyPond', bool(relevance)) + # Category:Categorized by DrTrigonBot def _addcat_BOT(self): # - ALWAYS - @@ -3611,7 +3552,7 @@ def log_output(self): # ColorRegions always applies here since there is at least 1 (THE average) color... - ignore = ['Properties', 'ColorAverage', 'ColorRegions', 'Geometry'] + ignore = ['Properties', 'Metadata', 'ColorAverage', 'ColorRegions', 'Geometry'] #if not self._existInformation(self._info): # information available? # information available? AND/OR category available? if not (self._existInformation(self._info, ignore = ignore) or self._result_check): @@ -3664,14 +3605,13 @@ return u"\n".join( ret ) def clean_cache(self): -# if os.path.exists(self.image_path): -# os.remove( self.image_path ) -# if os.path.exists(self.image_path_JPEG): -# os.remove( self.image_path_JPEG ) -# #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") -# #if os.path.exists(image_path_new): -# # os.remove( image_path_new ) - pass + if os.path.exists(self.image_path): + os.remove( self.image_path ) + #if os.path.exists(self.image_path_JPEG): + # os.remove( self.image_path_JPEG ) + ##image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") + ##if os.path.exists(image_path_new): + ## os.remove( image_path_new ) # LOOK ALSO AT: checkimages.CatImagesBot.report def report(self): @@ -3799,40 +3739,19 @@ # gather data from all information interfaces def gatherFeatures(self): - self._info['Properties'] = [{'Format': u'-', 'Pages': 0}] - self._info['Metadata'] = [] - self._info['ColorAverage'] = [] - self._info['ColorRegions'] = [] - self._info['Faces'] = [] - self._info['OpticalCodes'] = [] - self._info['People'] = [] - self._info['Chessboard'] = [] - self._info['Text'] = [] - self._info['Streams'] = [] - self._info['Audio'] = [] - self._info['Legs'] = [] - self._info['Hands'] = [] - self._info['Torsos'] = [] - self._info['Ears'] = [] - self._info['Eyes'] = [] - self._info['Automobiles'] = [] - # split detection and extraction according to file types; JpegFile, ... TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) - with TypeFile(self.image_path, self.image_mime) as tf: - import copy - tf.__dict__.update(copy.deepcopy(self.__dict__)) + with TypeFile(self.image_path) as tf: + tf.image_mime = self.image_mime + tf.image = self.image for func in ['getProperties', 'getFeatures']: result = getattr(tf, func)() - if result: - self._info.update(result) + self._info.update(result) print self._info - print tf._info #print tf.__dict__ - self._info = tf._info self.image_size = tf.image_size - def _existInformation(self, info, ignore = ['Properties', 'ColorAverage']): + def _existInformation(self, info, ignore = ['Properties', 'Metadata', 'ColorAverage']): result = [] for item in info: if item in ignore: @@ -3846,6 +3765,11 @@ result = self._info['Properties'] return {'Properties': result} + def _filter_Metadata(self): + # >>> never drop <<< + result = self._info['Metadata'] + return {'Metadata': result} + def _filter_Faces(self): result = self._info['Faces'] if (len(result) < self._thrhld_group_size):

1 0

SVN: [11622] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 07 Jun '13

07 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11622 Revision: 11622 Author: drtrigon Date: 2013-06-07 18:21:31 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; format conversion in downloadImage and _detect_Properties_PIL re-aranged Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-07 14:53:04 UTC (rev 11621) +++ trunk/pywikipedia/catimages.py 2013-06-07 18:21:31 UTC (rev 11622) @@ -146,124 +146,36 @@ class UnknownFile(object): def __init__(self, filename, *args, **kwargs): - self.filename = filename + self.filename = filename + self.image_size = (None, None) + self._info = {} - def getProperties(self): - # check/look into the file by midnight commander (mc) - # https://pypi.python.org/pypi/hachoir-metadata - #self._detect_HeaderAndMetadata() + def __enter__(self): + return self - # get mime-type file-size, ... + def __exit__(self, type, value, traceback): pass + def getProperties(self): + result = {} + result.update( self._detect_HeaderAndMetadata() ) # Metadata + result.update( self._detect_Properties_PIL() ) # Properties + return result + def getFeatures(self): - pass + pywikibot.warning(u"File format '%s/%s' not supported (yet)!" % tuple(self.image_mime[:2])) + def _detect_HeaderAndMetadata(self): + # check/look into the file by midnight commander (mc) + # https://pypi.python.org/pypi/hachoir-metadata + return {} + def _detect_Properties_PIL(self): - """Retrieve as much file property info possible, especially the same - as commons does in order to compare if those libraries (ImageMagick, - ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - result = {'Format': u'-', 'Pages': 0} - if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' - # similar to PDF page count OR use BeautifulSoup - svgcountpages = re.compile("<page>") - pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + # get mime-type file-size, ... + return {} - #svg = rsvg.Handle(self.image_path) - # http://validator.w3.org/docs/api.html#libs - # http://pypi.python.org/pypi/py_w3c/ - vld = HTMLValidator() - valid = u'SVG' - try: - vld.validate(self.image.fileUrl()) - valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') - except urllib2.URLError: - pass - except ValidationFault: - pass - #print vld.errors, vld.warnings - - #self.image_size = (svg.props.width, svg.props.height) - - result = { 'Format': valid, - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - # may be set {{validSVG}} also or do something in bot template to - # recognize 'Format=SVG (valid)' ... - elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' - # http://code.activestate.com/recipes/496837-count-pdf-pages/ - #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) - rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 - pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) - - result = { 'Format': u'PDF', - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'x-xcf': # MIME: 'image/x-xcf; charset=binary' - result = { 'Format': u'%s' % self.image_mime[1].upper() } - # DO NOT use ImageMagick (identify) instead of PIL to get these info !! - elif (self.image_mime[0] == 'image') and \ - (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... - try: - i = Image.open(self.image_path) - except IOError: - pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') - return {'Properties': [result]} - - # http://mail.python.org/pipermail/image-sig/1999-May/000740.html - pc=0 # count number of pages - while True: - try: - i.seek(pc) - except EOFError: - break - pc+=1 - i.seek(0) # restore default - - # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles - # python-lcms (littlecms) may be freeimage library - #icc = i.app['APP2'] # jpeg - #icc = i.tag[34675] # tiff - #icc = re.sub('[^%s]'%string.printable, ' ', icc) - ## more image formats and more post-processing needed... - - #self.image_size = i.size - - result = { #'bands': i.getbands(), - #'bbox': i.getbbox(), - 'Format': i.format, - 'Mode': i.mode, - #'info': i.info, - #'stat': os.stat(self.image_path), - 'Palette': str(len(i.palette.palette)) if i.palette else u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' - # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - #print d - result['Format'] = u'%s' % d['format']['format_name'].upper() - elif self.image_mime[1] == 'midi': # MIME: 'audio/midi; charset=binary' - result['Format'] = u'%s' % self.image_mime[1].upper() - # djvu: python-djvulibre or python-djvu for djvu support - # http://pypi.python.org/pypi/python-djvulibre/0.3.9 - else: - pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') - return {'Properties': [result]} - - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) - - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return {'Properties': [result]} - - class JpegFile(UnknownFile): # for '_detect_Trained_CV' cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), @@ -280,12 +192,25 @@ # ('Hands' does not behave very well, in fact it detects any kind of skin and other things...) #(u'Aeroplanes', 'haarcascade_aeroplane.xml'),] # e.g. for 'Category:Unidentified aircraft' - def getProperties(self): - pass + def __init__(self, filename, *args, **kwargs): + UnknownFile.__init__(self, filename) - # Image size - self._detect_Properties_PIL() + self.image_filename = os.path.split(self.filename)[-1] + self.image_fileext = os.path.splitext(self.image_filename)[1] + self.image_path = os.path.join(scriptdir, ('cache/' + self.image_filename[-128:])) + self.image_path_JPEG = self.image_path + '.jpg' + self._convert() + + def __exit__(self, type, value, traceback): + if os.path.exists(self.image_path): + os.remove( self.image_path ) + if os.path.exists(self.image_path_JPEG): + os.remove( self.image_path_JPEG ) + #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") + #if os.path.exists(image_path_new): + # os.remove( image_path_new ) + def getFeatures(self): # Faces (extract EXIF data) self._detect_Faces_EXIF() @@ -336,6 +261,87 @@ # general file EXIF history information self._detect_History_EXIF() + # supports a lot of different file types thanks to PIL + def _convert(self): + try: + im = Image.open(self.image_path) # might be png, gif etc, for instance + #im.thumbnail(size, Image.ANTIALIAS) # size is 640x480 + im.convert('RGB').save(self.image_path_JPEG, "JPEG") + + self.image_size = im.size + except IOError, e: + if 'image file is truncated' in str(e): + # im object has changed due to exception raised + im.convert('RGB').save(self.image_path_JPEG, "JPEG") + + self.image_size = im.size + else: + try: + # since opencv might still work, try this as fall-back + img = cv2.imread( self.image_path, cv.CV_LOAD_IMAGE_COLOR ) + cv2.imwrite(self.image_path_JPEG, img) + + self.image_size = (img.shape[1], img.shape[0]) + except: + if os.path.exists(self.image_path_JPEG): + os.remove(self.image_path_JPEG) + self.image_path_JPEG = self.image_path + except: + self.image_path_JPEG = self.image_path + + # FULL TIFF support (e.g. group4) + # http://code.google.com/p/pylibtiff/ + + # MIME: 'image/jpeg; charset=binary', ... + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + try: + i = Image.open(self.image_path) + except IOError: + pywikibot.warning(u'unknown file type [JpegFile]') + return {'Properties': [result]} + + # http://mail.python.org/pipermail/image-sig/1999-May/000740.html + pc=0 # count number of pages + while True: + try: + i.seek(pc) + except EOFError: + break + pc+=1 + i.seek(0) # restore default + + # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles + # python-lcms (littlecms) may be freeimage library + #icc = i.app['APP2'] # jpeg + #icc = i.tag[34675] # tiff + #icc = re.sub('[^%s]'%string.printable, ' ', icc) + ## more image formats and more post-processing needed... + + #self.image_size = i.size + + result = { #'bands': i.getbands(), + #'bbox': i.getbbox(), + 'Format': i.format, + 'Mode': i.mode, + #'info': i.info, + #'stat': os.stat(self.image_path), + 'Palette': str(len(i.palette.palette)) if i.palette else u'-', + 'Pages': pc, } + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html def _detect_Faces_CV(self): @@ -2477,15 +2483,162 @@ class TiffFile(JpegFile): pass + class XcfFile(JpegFile): - pass + def _convert(self): + # Very few programs other than GIMP read XCF files. This is by design + # from the GIMP developers, the format is not really documented or + # supported as a general-purpose file format. + # Commons uses ImageMagick, thus we have EXACTLY THE SAME support! + # (can also be a drawback, e.g. when the library is buggy...) + proc = Popen("convert %s %s" % (self.image_path, self.image_path_JPEG), + shell=True, stderr=PIPE)#.stderr.read() + proc.wait() + if proc.returncode != 0: + raise ImportError("convert (ImageMagick) not found (may be other error occured)!") + elif proc.returncode: + self.image_path_JPEG = self.image_path + #data = Popen("identify -verbose info: %s" % self.image_path, + # shell=True, stderr=PIPE).stderr.read() + #print data + if not os.path.exists(self.image_path_JPEG): + # xcf can have more than 1 layer/page like gif, tiff, and movies... + self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') + self.image_size = Image.open(self.image_path_JPEG).size + + # MIME: 'image/x-xcf; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + result = { 'Format': u'%s' % self.image_mime[1].upper() } + # DO NOT use ImageMagick (identify) instead of PIL to get these info !! + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + class SvgFile(JpegFile): - pass + def _convert(self): + # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) + # (Mediawiki uses librsvg too: http://commons.wikimedia.org/wiki/SVG#SVGs_in_MediaWiki) + # http://stackoverflow.com/questions/6589358/convert-svg-to-png-in-python + # http://cairographics.org/pythoncairopil/ + # http://cairographics.org/pyrsvg/ + # http://stackoverflow.com/questions/9166400/convert-rgba-png-to-rgb-with-pil + try: + svg = rsvg.Handle(self.image_path) + img = cairo.ImageSurface(cairo.FORMAT_ARGB32, svg.props.width, svg.props.height) + ctx = cairo.Context(img) + svg.render_cairo(ctx) + #img.write_to_png("svg.png") + #Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), + # img.get_data(),"raw","RGBA",0,1).save(self.image_path_JPEG, "JPEG") + png = Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), + img.get_data(),"raw","RGBA",0,1) + background = Image.new("RGB", png.size, (255, 255, 255)) + background.paste(png, mask=png.split()[3]) # 3 is the alpha channel + background.save(self.image_path_JPEG, "JPEG") + self.image_size = (svg.props.width, svg.props.height) + except MemoryError: + self.image_path_JPEG = self.image_path + except SystemError: + self.image_path_JPEG = self.image_path + + # MIME: 'application/xml; charset=utf-8' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # similar to PDF page count OR use BeautifulSoup + svgcountpages = re.compile("<page>") + pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + + #svg = rsvg.Handle(self.image_path) + + # http://validator.w3.org/docs/api.html#libs + # http://pypi.python.org/pypi/py_w3c/ + vld = HTMLValidator() + valid = u'SVG' + try: + vld.validate(self.image.fileUrl()) + valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') + except urllib2.URLError: + pass + except ValidationFault: + pass + #print vld.errors, vld.warnings + + #self.image_size = (svg.props.width, svg.props.height) + + result = { 'Format': valid, + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + # may be set {{validSVG}} also or do something in bot template to + # recognize 'Format=SVG (valid)' ... + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + class PdfFile(JpegFile): - pass + def _convert(self): +# self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) + # PDF: support extract text and images + # (Mediawiki uses ghostscript: https://www.mediawiki.org/wiki/Extension:PdfHandler#Pre-requisites) + # http://vermeulen.ca/python-pdf.html + # http://code.activestate.com/recipes/511465-pure-python-pdf-to-text-converte… + # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… + if self.image_fileext == u'.pdf': + pass + + # MIME: 'application/pdf; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # http://code.activestate.com/recipes/496837-count-pdf-pages/ + #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) + rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 + pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) + + result = { 'Format': u'PDF', + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + #class DjvuFile(JpegFile): # pass @@ -2498,6 +2651,27 @@ # general audio feature extraction # self._detect_AudioFeatures_YAAFE() + # MIME: 'application/ogg; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + #print d + result['Format'] = u'%s' % d['format']['format_name'].upper() + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + def _detect_Streams_FFMPEG(self): self._info['Streams'] = [] @@ -2696,12 +2870,6 @@ class MidiFile(UnknownFile): - def getProperties(self): - result = {} - result.update( self._detect_HeaderAndMetadata() ) # Metadata - result.update( self._detect_Properties_PIL() ) # Properties - return result - def getFeatures(self): result = {} result.update( self._detect_AudioFeatures_MUSIC21() ) # Audio @@ -2739,6 +2907,24 @@ self._info['Metadata'] = [result] return {'Metadata': [result]} + # MIME: 'audio/midi; charset=binary' + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + + result['Format'] = u'%s' % self.image_mime[1].upper() + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + # midi audio feature extraction def _detect_AudioFeatures_MUSIC21(self): # skip file formats not supported @@ -2805,6 +2991,8 @@ ( 'image', 'x-xcf'): XcfFile, ( 'image', 'svg+xml'): SvgFile, ('application', 'pdf'): PdfFile, +# djvu: python-djvulibre or python-djvu for djvu support +# http://pypi.python.org/pypi/python-djvulibre/0.3.9 # ( 'image', 'vnd.djvu'): DjvuFile, ('application', 'ogg'): OggFile, ( 'audio', 'midi'): MidiFile,} @@ -3257,12 +3445,10 @@ #print self.image_path pywikibot.output(u'Processing media %s ...' % self.image.title(asLink=True)) - self.image_filename = os.path.split(self.image.fileUrl())[-1] - self.image_fileext = os.path.splitext(self.image_filename)[1] - self.image_path = urllib2.quote(os.path.join(scriptdir, ('cache/' + self.image_filename[-128:]))) - - self.image_path_JPEG = self.image_path + u'.jpg' - + image_filename = os.path.split(self.image.fileUrl())[-1] + image_fileext = os.path.splitext(image_filename)[1] + self.image_path = urllib2.quote(os.path.join(scriptdir, ('cache/' + image_filename[-128:]))) + self._wikidata = self.image._latestInfo # all info wikimedia got from content (mime, sha1, ...) #print self._wikidata #print self._wikidata['mime'] @@ -3270,7 +3456,7 @@ #print self._wikidata['metadata'] #for item in self._wikidata['metadata']: # print item['name'], item['value'] - + if not os.path.exists(self.image_path): pywikibot.get_throttle() f_url, data = self.site.getUrl(self.image.fileUrl(), no_hostname=True, @@ -3279,7 +3465,7 @@ # (allows to re-read from back_response) data = f_url.read() del f_url # free some memory (no need to keep a copy...) - + f = open(self.image_path, 'wb') f.write( data ) f.close() @@ -3290,95 +3476,9 @@ self.image_mime = re.split('[/;\s]', m.file(self.image_path)) #self.image_size = (None, None) mime = mimetypes.guess_all_extensions('%s/%s' % tuple(self.image_mime[0:2])) - if mime and (self.image_fileext.lower() not in mime): + if mime and (image_fileext.lower() not in mime): pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) - # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) - # (Mediawiki uses librsvg too: http://commons.wikimedia.org/wiki/SVG#SVGs_in_MediaWiki) - # http://stackoverflow.com/questions/6589358/convert-svg-to-png-in-python - # http://cairographics.org/pythoncairopil/ - # http://cairographics.org/pyrsvg/ - # http://stackoverflow.com/questions/9166400/convert-rgba-png-to-rgb-with-pil - self.image_size = (None, None) - if self.image_fileext == u'.svg': - try: - svg = rsvg.Handle(self.image_path) - img = cairo.ImageSurface(cairo.FORMAT_ARGB32, svg.props.width, svg.props.height) - ctx = cairo.Context(img) - svg.render_cairo(ctx) - #img.write_to_png("svg.png") - #Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), - # img.get_data(),"raw","RGBA",0,1).save(self.image_path_JPEG, "JPEG") - png = Image.frombuffer("RGBA",( img.get_width(),img.get_height() ), - img.get_data(),"raw","RGBA",0,1) - background = Image.new("RGB", png.size, (255, 255, 255)) - background.paste(png, mask=png.split()[3]) # 3 is the alpha channel - background.save(self.image_path_JPEG, "JPEG") - - self.image_size = (svg.props.width, svg.props.height) - except MemoryError: - self.image_path_JPEG = self.image_path - except SystemError: - self.image_path_JPEG = self.image_path - elif self.image_fileext == u'.xcf': - # Very few programs other than GIMP read XCF files. This is by design - # from the GIMP developers, the format is not really documented or - # supported as a general-purpose file format. - # Commons uses ImageMagick, thus we have EXACTLY THE SAME support! - # (can also be a drawback, e.g. when the library is buggy...) - proc = Popen("convert %s %s" % (self.image_path, self.image_path_JPEG), - shell=True, stderr=PIPE)#.stderr.read() - proc.wait() - if proc.returncode == 127: - raise ImportError("convert (ImageMagick) not found!") - elif proc.returncode: - self.image_path_JPEG = self.image_path - - #data = Popen("identify -verbose info: %s" % self.image_path, - # shell=True, stderr=PIPE).stderr.read() - #print data - if not os.path.exists(self.image_path_JPEG): - # xcf can have more than 1 layer/page like gif, tiff, and movies... - self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') - self.image_size = Image.open(self.image_path_JPEG).size - else: - try: - im = Image.open(self.image_path) # might be png, gif etc, for instance - #im.thumbnail(size, Image.ANTIALIAS) # size is 640x480 - im.convert('RGB').save(self.image_path_JPEG, "JPEG") - - self.image_size = im.size - except IOError, e: - if 'image file is truncated' in str(e): - # im object has changed due to exception raised - im.convert('RGB').save(self.image_path_JPEG, "JPEG") - - self.image_size = im.size - else: - try: - # since opencv might still work, try this as fall-back - img = cv2.imread( self.image_path, cv.CV_LOAD_IMAGE_COLOR ) - cv2.imwrite(self.image_path_JPEG, img) - - self.image_size = (img.shape[1], img.shape[0]) - except: - if os.path.exists(self.image_path_JPEG): - os.remove(self.image_path_JPEG) - self.image_path_JPEG = self.image_path - except: - self.image_path_JPEG = self.image_path - - # PDF: support extract text and images - # (Mediawiki uses ghostscript: https://www.mediawiki.org/wiki/Extension:PdfHandler#Pre-requisites) - # http://vermeulen.ca/python-pdf.html - # http://code.activestate.com/recipes/511465-pure-python-pdf-to-text-converte… - # http://stackoverflow.com/questions/25665/python-module-for-converting-pdf-t… - if self.image_fileext == u'.pdf': - pass - - # FULL TIFF support (e.g. group4) - # http://code.google.com/p/pylibtiff/ - # LOOK ALSO AT: checkimages.CatImagesBot.checkStep # (and category scripts/bots too...) def checkStep(self): @@ -3564,13 +3664,14 @@ return u"\n".join( ret ) def clean_cache(self): - if os.path.exists(self.image_path): - os.remove( self.image_path ) - if os.path.exists(self.image_path_JPEG): - os.remove( self.image_path_JPEG ) - #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") - #if os.path.exists(image_path_new): - # os.remove( image_path_new ) +# if os.path.exists(self.image_path): +# os.remove( self.image_path ) +# if os.path.exists(self.image_path_JPEG): +# os.remove( self.image_path_JPEG ) +# #image_path_new = self.image_path_JPEG.replace(u"cache/", u"cache/0_DETECTED_") +# #if os.path.exists(image_path_new): +# # os.remove( image_path_new ) + pass # LOOK ALSO AT: checkimages.CatImagesBot.report def report(self): @@ -3718,17 +3819,18 @@ # split detection and extraction according to file types; JpegFile, ... TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) - tf = TypeFile(self.image_path) - import copy - tf.__dict__.update(copy.deepcopy(self.__dict__)) - for func in ['getProperties', 'getFeatures']: - result = getattr(tf, func)() - if result: - self._info.update(result) -# print self._info -# print tf._info - #print tf.__dict__ - self._info = tf._info + with TypeFile(self.image_path, self.image_mime) as tf: + import copy + tf.__dict__.update(copy.deepcopy(self.__dict__)) + for func in ['getProperties', 'getFeatures']: + result = getattr(tf, func)() + if result: + self._info.update(result) + print self._info + print tf._info + #print tf.__dict__ + self._info = tf._info + self.image_size = tf.image_size def _existInformation(self, info, ignore = ['Properties', 'ColorAverage']): result = []

1 0

SVN: [11621] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 07 Jun '13

07 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11621 Revision: 11621 Author: drtrigon Date: 2013-06-07 14:53:04 +0000 (Fri, 07 Jun 2013) Log Message: ----------- improvement; start of code structure clean-up Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-06 21:19:06 UTC (rev 11620) +++ trunk/pywikipedia/catimages.py 2013-06-07 14:53:04 UTC (rev 11621) @@ -143,7 +143,199 @@ # all detection and recognition methods - bindings to other classes, modules and libs -class FileData(object): + +class UnknownFile(object): + def __init__(self, filename, *args, **kwargs): + self.filename = filename + self._info = {} + + def getProperties(self): + # check/look into the file by midnight commander (mc) + # https://pypi.python.org/pypi/hachoir-metadata + #self._detect_HeaderAndMetadata() + + # get mime-type file-size, ... + pass + + def getFeatures(self): + pass + + def _detect_Properties_PIL(self): + """Retrieve as much file property info possible, especially the same + as commons does in order to compare if those libraries (ImageMagick, + ...) are buggy (thus explicitely use other software for independence)""" + #self.image_size = (None, None) + result = {'Format': u'-', 'Pages': 0} + if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' + # similar to PDF page count OR use BeautifulSoup + svgcountpages = re.compile("<page>") + pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) + + #svg = rsvg.Handle(self.image_path) + + # http://validator.w3.org/docs/api.html#libs + # http://pypi.python.org/pypi/py_w3c/ + vld = HTMLValidator() + valid = u'SVG' + try: + vld.validate(self.image.fileUrl()) + valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') + except urllib2.URLError: + pass + except ValidationFault: + pass + #print vld.errors, vld.warnings + + #self.image_size = (svg.props.width, svg.props.height) + + result = { 'Format': valid, + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + # may be set {{validSVG}} also or do something in bot template to + # recognize 'Format=SVG (valid)' ... + elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' + # http://code.activestate.com/recipes/496837-count-pdf-pages/ + #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) + rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 + pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) + + result = { 'Format': u'PDF', + 'Mode': u'-', + 'Palette': u'-', + 'Pages': pc, } + elif self.image_mime[1] == 'x-xcf': # MIME: 'image/x-xcf; charset=binary' + result = { 'Format': u'%s' % self.image_mime[1].upper() } + # DO NOT use ImageMagick (identify) instead of PIL to get these info !! + elif (self.image_mime[0] == 'image') and \ + (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... + try: + i = Image.open(self.image_path) + except IOError: + pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') + return {'Properties': [result]} + + # http://mail.python.org/pipermail/image-sig/1999-May/000740.html + pc=0 # count number of pages + while True: + try: + i.seek(pc) + except EOFError: + break + pc+=1 + i.seek(0) # restore default + + # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles + # python-lcms (littlecms) may be freeimage library + #icc = i.app['APP2'] # jpeg + #icc = i.tag[34675] # tiff + #icc = re.sub('[^%s]'%string.printable, ' ', icc) + ## more image formats and more post-processing needed... + + #self.image_size = i.size + + result = { #'bands': i.getbands(), + #'bbox': i.getbbox(), + 'Format': i.format, + 'Mode': i.mode, + #'info': i.info, + #'stat': os.stat(self.image_path), + 'Palette': str(len(i.palette.palette)) if i.palette else u'-', + 'Pages': pc, } + elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' + # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + #print d + result['Format'] = u'%s' % d['format']['format_name'].upper() + elif self.image_mime[1] == 'midi': # MIME: 'audio/midi; charset=binary' + result['Format'] = u'%s' % self.image_mime[1].upper() + # djvu: python-djvulibre or python-djvu for djvu support + # http://pypi.python.org/pypi/python-djvulibre/0.3.9 + else: + pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') + return {'Properties': [result]} + + result['Dimensions'] = self.image_size + result['Filesize'] = os.path.getsize(self.image_path) + result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) + + #self._info['Properties'] = [result] + self._info['Properties'][0].update(result) + return {'Properties': [result]} + + +class JpegFile(UnknownFile): + # for '_detect_Trained_CV' + cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), + (u'Torsos', 'haarcascade_upperbody.xml'), + (u'Ears', 'haarcascade_mcs_leftear.xml'), + (u'Ears', 'haarcascade_mcs_rightear.xml'), + (u'Eyes', 'haarcascade_lefteye_2splits.xml'), # (http://yushiqi.cn/research/eyedetection) + (u'Eyes', 'haarcascade_righteye_2splits.xml'), # (http://yushiqi.cn/research/eyedetection) + #externals/opencv/haarcascades/haarcascade_mcs_lefteye.xml + #externals/opencv/haarcascades/haarcascade_mcs_righteye.xml + # (others include indifferent (left and/or right) and pair) + (u'Automobiles', 'cars3.xml'), # http://www.youtube.com/watch?v=c4LobbqeKZc + (u'Hands', '1256617233-2-haarcascade-hand.xml', 300.),] # http://www.andol.info/ + # ('Hands' does not behave very well, in fact it detects any kind of skin and other things...) + #(u'Aeroplanes', 'haarcascade_aeroplane.xml'),] # e.g. for 'Category:Unidentified aircraft' + + def getProperties(self): + pass + + # Image size + self._detect_Properties_PIL() + + def getFeatures(self): + # Faces (extract EXIF data) + self._detect_Faces_EXIF() + # Faces and eyes (opencv pre-trained haar) + self._detect_Faces_CV() +# TODO: test and use or switch off + # Face via Landmark(s) +# self._detect_FaceLandmark_xBOB() + # exclude duplicates (CV and EXIF) + faces = [item['Position'] for item in self._info['Faces']] + for i in self._util_merge_Regions(faces)[1]: + del self._info['Faces'][i] + + # Segments and colors + self._detect_SegmentColors_JSEGnPIL() + # Average color + self._detect_AverageColor_PILnCV() + + # People/Pedestrian (opencv pre-trained hog and haarcascade) + self._detect_People_CV() + + # Geometric object (opencv hough line, circle, edges, corner, ...) + self._detect_Geometry_CV() + + # general (opencv pre-trained, third-party and self-trained haar + # and cascade) classification + # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… + for cf in self.cascade_files: + self._detect_Trained_CV(*cf) + + # optical and other text recognition (tesseract & ocropus, ...) + self._detect_EmbeddedText_poppler() +# self._recognize_OpticalText_ocropus() + # (may be just classify as 'contains text', may be store text, e.g. to wikisource) + + # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) + self._recognize_OpticalCodes_dmtxNzbar() + + # Chessboard (opencv reference detector) + self._detect_Chessboard_CV() + + # general (self-trained) detection WITH classification + # BoW: uses feature detection (SIFT, SURF, ...) AND classification (SVM, ...) +# self._detectclassify_ObjectAll_CV() + # Wavelet: uses wavelet transformation AND classification (machine learning) +# self._detectclassify_ObjectAll_PYWT() + + # general file EXIF history information + self._detect_History_EXIF() + # .../opencv/samples/c/facedetect.cpp # http://opencv.willowgarage.com/documentation/python/genindex.html def _detect_Faces_CV(self): @@ -157,7 +349,8 @@ # http://www.cognotics.com/opencv/servo_2007_series/part_4/index.html # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # https://code.ros.org/trac/opencv/browser/trunk/opencv_extra/testdata/gpu/ha… @@ -699,7 +892,8 @@ self._info['Geometry'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return result = self._util_get_Geometry_CVnSCIPY() @@ -970,7 +1164,8 @@ self._info['ColorRegions'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return try: @@ -1032,7 +1227,8 @@ self._info['ColorAverage'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return try: @@ -1050,109 +1246,6 @@ self._info['ColorAverage'] = [result] return - def _detect_Properties_PIL(self): - """Retrieve as much file property info possible, especially the same - as commons does in order to compare if those libraries (ImageMagick, - ...) are buggy (thus explicitely use other software for independence)""" - #self.image_size = (None, None) - self._info['Properties'] = [{'Format': u'-', 'Pages': 0}] - if self.image_fileext == u'.svg': # MIME: 'application/xml; charset=utf-8' - # similar to PDF page count OR use BeautifulSoup - svgcountpages = re.compile("<page>") - pc = len(svgcountpages.findall( file(self.image_path,"r").read() )) - - #svg = rsvg.Handle(self.image_path) - - # http://validator.w3.org/docs/api.html#libs - # http://pypi.python.org/pypi/py_w3c/ - vld = HTMLValidator() - valid = u'SVG' - try: - vld.validate(self.image.fileUrl()) - valid = (u'Valid SVG' if vld.result.validity == 'true' else u'Invalid SVG') - except urllib2.URLError: - pass - except ValidationFault: - pass - #print vld.errors, vld.warnings - - #self.image_size = (svg.props.width, svg.props.height) - - result = { 'Format': valid, - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - # may be set {{validSVG}} also or do something in bot template to - # recognize 'Format=SVG (valid)' ... - elif self.image_mime[1] == 'pdf': # MIME: 'application/pdf; charset=binary' - # http://code.activestate.com/recipes/496837-count-pdf-pages/ - #rxcountpages = re.compile(r"$\s*/Type\s*/Page[/\s]", re.MULTILINE|re.DOTALL) - rxcountpages = re.compile(r"/Type\s*/Page([^s]|$)", re.MULTILINE|re.DOTALL) # PDF v. 1.3,1.4,1.5,1.6 - pc = len(rxcountpages.findall( file(self.image_path,"rb").read() )) - - result = { 'Format': u'PDF', - 'Mode': u'-', - 'Palette': u'-', - 'Pages': pc, } - elif (self.image_mime[0] == 'image') and \ - (self.image_mime[1] != 'vnd.djvu'): # MIME: 'image/jpeg; charset=binary', ... - try: - i = Image.open(self.image_path) - except IOError: - pywikibot.warning(u'unknown (image) file type [_detect_Properties_PIL]') - return - - # http://mail.python.org/pipermail/image-sig/1999-May/000740.html - pc=0 # count number of pages - while True: - try: - i.seek(pc) - except EOFError: - break - pc+=1 - i.seek(0) # restore default - - # http://grokbase.com/t/python/image-sig/082psaxt6k/embedded-icc-profiles - # python-lcms (littlecms) may be freeimage library - #icc = i.app['APP2'] # jpeg - #icc = i.tag[34675] # tiff - #icc = re.sub('[^%s]'%string.printable, ' ', icc) - ## more image formats and more post-processing needed... - - #self.image_size = i.size - - result = { #'bands': i.getbands(), - #'bbox': i.getbbox(), - 'Format': i.format, - 'Mode': i.mode, - #'info': i.info, - #'stat': os.stat(self.image_path), - 'Palette': str(len(i.palette.palette)) if i.palette else u'-', - 'Pages': pc, } - elif self.image_mime[1] == 'ogg': # MIME: 'application/ogg; charset=binary' - # 'ffprobe' (ffmpeg); audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - #print d - result = { 'Format': u'%s' % d['format']['format_name'].upper() } - elif self.image_mime[0] == 'audio': # MIME: 'audio/midi; charset=binary' - result = {} - # djvu: python-djvulibre or python-djvu for djvu support - # http://pypi.python.org/pypi/python-djvulibre/0.3.9 - #elif self.image_fileext == u'.xcf' - # result = {} - # # DO NOT use ImageMagick (identify) instead of PIL to get these info !! - else: - pywikibot.warning(u'unknown (generic) file type [_detect_Properties_PIL]') - return - - result['Dimensions'] = self.image_size - result['Filesize'] = os.path.getsize(self.image_path) - result['MIME'] = u'%s/%s' % tuple(self.image_mime[:2]) - - #self._info['Properties'] = [result] - self._info['Properties'][0].update(result) - return - # http://stackoverflow.com/questions/2270874/image-color-detection-using-pyth… # https://gist.github.com/1246268 # colormath-1.0.8/examples/delta_e.py, colormath-1.0.8/examples/conversions.py @@ -1413,7 +1506,8 @@ self._info[info_desc] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # http://tutorial-haartraining.googlecode.com/svn/trunk/data/haarcascades/ @@ -1634,7 +1728,8 @@ self._info['OpticalCodes'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return # DataMatrix @@ -1734,7 +1829,8 @@ self._info['Chessboard'] = [] # skip file formats not supported (yet?) - if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']): + if (self.image_mime[1] in ['ogg', 'pdf', 'vnd.djvu']) or \ + (self.image_mime[0] in ['audio']): return scale = 1. @@ -2306,45 +2402,6 @@ return self._buffer_FFMPEG - def _detect_Streams_FFMPEG(self): - self._info['Streams'] = [] - - # skip file formats that interfere and can cause strange results (pdf is oga?!) - # or file formats not supported (yet?) - if (self.image_mime[1] in ['pdf']) or (self.image_fileext in [u'.svg']): - return - - # audio and video streams files (ogv, oga, ...) - d = self._util_get_DataStreams_FFMPEG() - if not d: - return - - result = [] - for s in d['streams']: - #print s - if (s["codec_type"] == "video"): - rate = s["avg_frame_rate"] - dim = (int(s["width"]), int(s["height"])) - #asp = s["display_aspect_ratio"] - elif (s["codec_type"] == "audio"): -# switch this part off since 'ffprobe' (on toolserver) is too old -# rate = u'%s/%s/%s' % (s["channels"], s["sample_fmt"], s["sample_rate"]) - rate = u'%s/%s/%s' % (s["channels"], u'-', int(float(s["sample_rate"]))) - dim = None - elif (s["codec_type"] == "data"): - rate = None - dim = None - - result.append({ 'ID': int(s["index"]) + 1, - 'Format': u'%s/%s' % (s["codec_type"], s.get("codec_name",u'?')), - 'Rate': rate or u'-', - 'Dimensions': dim or (None, None), - }) - - if 'image' not in d["format"]["format_name"]: - self._info['Streams'] = result - return - def _util_merge_Regions(self, regs, sub=False, overlap=False, close=False): # sub=False, overlap=False, close=False ; level 0 ; similar regions, similar position (default) # sub=True, overlap=False, close=False ; level 1 ; region contained in other, any shape/size @@ -2410,70 +2467,74 @@ return (regs, drop) - def _detect_AudioFeatures_MUSIC21(self): - # skip file formats not supported - if (self.image_mime[1] not in ['midi']): - return - self._info['Audio'] = [] +class PngFile(JpegFile): + pass - import _music21 as music21 +class GifFile(JpegFile): + pass - #audiofile = '/home/ursin/Desktop/3_Ships.mid' - audiofile = self.image_path +class TiffFile(JpegFile): + pass - #music21.features.jSymbolic.getCompletionStats() - try: - #s = music21.converter.ConverterMidi() - #s.parseFile(audiofile) - #s = s.stream - s = music21.midi.translate.midiFilePathToStream(audiofile) - except music21.midi.base.MidiException: - pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') +class XcfFile(JpegFile): + pass + +class SvgFile(JpegFile): + pass + +class PdfFile(JpegFile): + pass + +#class DjvuFile(JpegFile): +# pass + + +class OggFile(JpegFile): + def getFeatures(self): + # general handling of all audio and video formats + self._detect_Streams_FFMPEG() + + # general audio feature extraction +# self._detect_AudioFeatures_YAAFE() + + def _detect_Streams_FFMPEG(self): + self._info['Streams'] = [] + + # skip file formats that interfere and can cause strange results (pdf is oga?!) + # or file formats not supported (yet?) + if (self.image_mime[1] in ['pdf']) or (self.image_fileext in [u'.svg']): return - #fs = music21.features.jSymbolic.extractorsById - #for k in fs: - # for i in range(len(fs[k])): - # if fs[k][i] is not None: - # n = fs[k][i].__name__ - # if fs[k][i] not in music21.features.jSymbolic.featureExtractors: - # n += " (not implemented)" - # print k, i, n - # else: - # fe = fs[k][i](s) - # print k, i, n, - # try: - # f = fe.extract() - # print f.name, f.vector - # except AttributeError: - # print "ERROR" - data = {} - for item in ['MostCommonPitchFeature', - 'ImportanceOfBassRegisterFeature', - 'ImportanceOfMiddleRegisterFeature', - 'ImportanceOfHighRegisterFeature', - 'AverageNoteDurationFeature', - 'MaximumNoteDurationFeature', - #'DurationFeature', - 'InitialTempoFeature', - 'MaximumNumberOfIndependentVoicesFeature', - 'AverageNumberOfIndependentVoicesFeature',]: - fe = getattr(music21.features.jSymbolic, item)(s) - f = fe.extract() - #data[f.name] = f.vector[0] - data[item.replace('Feature', '')] = f.vector[0] - #print s.duration - data['Duration'] = s.highestTime - #print s.offsetMap - #print s.measureOffsetMap() - data['Metadata'] = s.metadata - data['Lyrics'] = s.lyrics() - #print s.seconds - #print s.secondsMap + # audio and video streams files (ogv, oga, ...) + d = self._util_get_DataStreams_FFMPEG() + if not d: + return - self._info['Audio'] = [data] -# print self._info['Audio'] + result = [] + for s in d['streams']: + #print s + if (s["codec_type"] == "video"): + rate = s["avg_frame_rate"] + dim = (int(s["width"]), int(s["height"])) + #asp = s["display_aspect_ratio"] + elif (s["codec_type"] == "audio"): +# switch this part off since 'ffprobe' (on toolserver) is too old +# rate = u'%s/%s/%s' % (s["channels"], s["sample_fmt"], s["sample_rate"]) + rate = u'%s/%s/%s' % (s["channels"], u'-', int(float(s["sample_rate"]))) + dim = None + elif (s["codec_type"] == "data"): + rate = None + dim = None + + result.append({ 'ID': int(s["index"]) + 1, + 'Format': u'%s/%s' % (s["codec_type"], s.get("codec_name",u'?')), + 'Rate': rate or u'-', + 'Dimensions': dim or (None, None), + }) + + if 'image' not in d["format"]["format_name"]: + self._info['Streams'] = result return def _detect_AudioFeatures_YAAFE(self): @@ -2634,10 +2695,125 @@ return +class MidiFile(UnknownFile): + def getProperties(self): + result = {} + result.update( self._detect_HeaderAndMetadata() ) # Metadata + result.update( self._detect_Properties_PIL() ) # Properties + return result + + def getFeatures(self): + result = {} + result.update( self._detect_AudioFeatures_MUSIC21() ) # Audio + return result + + def _detect_HeaderAndMetadata(self): + result = {} + + # extract data from midi file + # http://valentin.dasdeck.com/midi/midifile.htm + # http://stackoverflow.com/questions/3943149/reading-and-interpreting-data-fr… + ba = bytearray(open(self.filename, 'rb').read()) + i = -1 + for key, data in [('Text', '\x01'), ('Copyright', '\x02'), ('Lyrics', '\x05')]: + result[key] = [] + while True: + i = ba.find('\xff%s' % data, i+1) + if i < 0: # something found? + break + e = (i+3+ba[i+2]) + if ba[e] != 0: # length match with string end (00)? + e = ba.find('\x00', (i+3+ba[i+2])) + result[key].append(ba[i+3:e].decode('latin-1').strip()) + result[key] = u'\n'.join(result[key]) + + import _music21 as music21 + try: + s = music21.converter.parse(self.filename) + if s.metadata: + pywikibot.output(unicode(s.metadata)) + result['Metadata'].update(s.metadata) + except music21.midi.base.MidiException: + pass + + self._info['Metadata'] = [result] + return {'Metadata': [result]} + + # midi audio feature extraction + def _detect_AudioFeatures_MUSIC21(self): + # skip file formats not supported + if (self.image_mime[1] not in ['midi']): + return + + import _music21 as music21 + + #audiofile = '/home/ursin/Desktop/3_Ships.mid' + audiofile = self.image_path + + #music21.features.jSymbolic.getCompletionStats() + try: + #s = music21.midi.translate.midiFilePathToStream(audiofile) + s = music21.converter.parse(audiofile) + except music21.midi.base.MidiException: + pywikibot.warning(u'unknown file type [_detect_AudioFeatures_MUSIC21]') + return + + #fs = music21.features.jSymbolic.extractorsById + #for k in fs: + # for i in range(len(fs[k])): + # if fs[k][i] is not None: + # n = fs[k][i].__name__ + # if fs[k][i] not in music21.features.jSymbolic.featureExtractors: + # n += " (not implemented)" + # print k, i, n + # else: + # fe = fs[k][i](s) + # print k, i, n, + # try: + # f = fe.extract() + # print f.name, f.vector + # except AttributeError: + # print "ERROR" + data = {'RegisterImportance': (music21.features.jSymbolic.ImportanceOfBassRegisterFeature(s).extract().vector[0], + music21.features.jSymbolic.ImportanceOfMiddleRegisterFeature(s).extract().vector[0], + music21.features.jSymbolic.ImportanceOfHighRegisterFeature(s).extract().vector[0],), + 'NoteDuration': (music21.features.jSymbolic.AverageNoteDurationFeature(s).extract().vector[0], + music21.features.jSymbolic.MaximumNoteDurationFeature(s).extract().vector[0],), + 'IndependentVoices': (music21.features.jSymbolic.AverageNumberOfIndependentVoicesFeature(s).extract().vector[0], + music21.features.jSymbolic.MaximumNumberOfIndependentVoicesFeature(s).extract().vector[0],), + 'MostCommonPitch': music21.features.jSymbolic.MostCommonPitchFeature(s).extract().vector[0], + 'Tempo': music21.features.jSymbolic.InitialTempoFeature(s).extract().vector[0], + 'Duration': s.highestTime, + #'Metadata': s.metadata if s.metadata else u'', + 'Lyrics': s.lyrics(recurse=True) if s.lyrics(recurse=True) else u'',} + #print music21.text.assembleLyrics(s) + #print s.duration + #print s.offsetMap + #print s.measureOffsetMap() + #print s.seconds + #print s.secondsMap + + self._info['Audio'] += [data] + return {'Audio': [data]} + + +FILETYPES = { '*': UnknownFile, + ( 'image', 'jpeg'): JpegFile, + ( 'image', 'png'): PngFile, + ( 'image', 'gif'): GifFile, + ( 'image', 'tiff'): TiffFile, + ( 'image', 'x-xcf'): XcfFile, + ( 'image', 'svg+xml'): SvgFile, + ('application', 'pdf'): PdfFile, +# ( 'image', 'vnd.djvu'): DjvuFile, + ('application', 'ogg'): OggFile, + ( 'audio', 'midi'): MidiFile,} + + # all classification and categorization methods and definitions - default variation # use simplest classification I can think of (self-made) and do categorization # mostly based on filtered/reported features -class CatImages_Default(FileData): +class CatImages_Default(object): #ignore = [] ignore = ['color'] @@ -2854,6 +3030,21 @@ return (u'Graphics', bool(relevance)) +# # Category:MIDI files created with GNU LilyPond +# def _cat_audio_MIDIfilescreatedwithGNULilyPond(self): +# # find metadata in extracted data +# print [item.strip() for item in re.findall('Generated .*?\n', u'\n'.join(result['Text']))] +# #u"Cr'eateur: GNU LilyPond 2.0.1" +# import dateutil.parser +# dates = [] +# for line in result['Text']: +# # http://stackoverflow.com/questions/3276180/extracting-date-from-a-string-in… +# try: +# dates.append(dateutil.parser.parse(line, fuzzy=True).isoformat(' ').decode('utf-8')) +# except ValueError: +# pass +# print dates + # Category:Categorized by DrTrigonBot def _addcat_BOT(self): # - ALWAYS - @@ -3099,7 +3290,7 @@ self.image_mime = re.split('[/;\s]', m.file(self.image_path)) #self.image_size = (None, None) mime = mimetypes.guess_all_extensions('%s/%s' % tuple(self.image_mime[0:2])) - if self.image_fileext.lower() not in mime: + if mime and (self.image_fileext.lower() not in mime): pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime) # SVG: rasterize the SVG to bitmap (MAY BE GET FROM WIKI BY DOWNLOAD?...) @@ -3146,6 +3337,9 @@ #data = Popen("identify -verbose info: %s" % self.image_path, # shell=True, stderr=PIPE).stderr.read() #print data + if not os.path.exists(self.image_path_JPEG): + # xcf can have more than 1 layer/page like gif, tiff, and movies... + self.image_path_JPEG = self.image_path_JPEG.replace('.jpg', '-0.jpg') self.image_size = Image.open(self.image_path_JPEG).size else: try: @@ -3168,6 +3362,8 @@ self.image_size = (img.shape[1], img.shape[0]) except: + if os.path.exists(self.image_path_JPEG): + os.remove(self.image_path_JPEG) self.image_path_JPEG = self.image_path except: self.image_path_JPEG = self.image_path @@ -3502,69 +3698,38 @@ # gather data from all information interfaces def gatherFeatures(self): - # Image size - self._detect_Properties_PIL() - + self._info['Properties'] = [{'Format': u'-', 'Pages': 0}] + self._info['Metadata'] = [] + self._info['ColorAverage'] = [] + self._info['ColorRegions'] = [] self._info['Faces'] = [] - # Faces (extract EXIF data) - self._detect_Faces_EXIF() - # Faces and eyes (opencv pre-trained haar) - self._detect_Faces_CV() -# TODO: test and use or switch off - # Face via Landmark(s) -# self._detect_FaceLandmark_xBOB() - # exclude duplicates (CV and EXIF) - faces = [item['Position'] for item in self._info['Faces']] - for i in self._util_merge_Regions(faces)[1]: - del self._info['Faces'][i] + self._info['OpticalCodes'] = [] + self._info['People'] = [] + self._info['Chessboard'] = [] + self._info['Text'] = [] + self._info['Streams'] = [] + self._info['Audio'] = [] + self._info['Legs'] = [] + self._info['Hands'] = [] + self._info['Torsos'] = [] + self._info['Ears'] = [] + self._info['Eyes'] = [] + self._info['Automobiles'] = [] - # Segments and colors - self._detect_SegmentColors_JSEGnPIL() - # Average color - self._detect_AverageColor_PILnCV() + # split detection and extraction according to file types; JpegFile, ... + TypeFile = FILETYPES.get(tuple(self.image_mime[:2]), FILETYPES['*']) + tf = TypeFile(self.image_path) + import copy + tf.__dict__.update(copy.deepcopy(self.__dict__)) + for func in ['getProperties', 'getFeatures']: + result = getattr(tf, func)() + if result: + self._info.update(result) +# print self._info +# print tf._info + #print tf.__dict__ + self._info = tf._info - # People/Pedestrian (opencv pre-trained hog and haarcascade) - self._detect_People_CV() - - # Geometric object (opencv hough line, circle, edges, corner, ...) - self._detect_Geometry_CV() - - # general (opencv pre-trained, third-party and self-trained haar - # and cascade) classification - # http://www.computer-vision-software.com/blog/2009/11/faq-opencv-haartrainin… - for cf in self.cascade_files: - self._detect_Trained_CV(*cf) - - # optical and other text recognition (tesseract & ocropus, ...) - self._detect_EmbeddedText_poppler() -# self._recognize_OpticalText_ocropus() - # (may be just classify as 'contains text', may be store text, e.g. to wikisource) - - # barcode and Data Matrix recognition (libdmtx/pydmtx, zbar, gocr?) - self._recognize_OpticalCodes_dmtxNzbar() - - # Chessboard (opencv reference detector) - self._detect_Chessboard_CV() - - # general (self-trained) detection WITH classification - # BoW: uses feature detection (SIFT, SURF, ...) AND classification (SVM, ...) -# self._detectclassify_ObjectAll_CV() - # Wavelet: uses wavelet transformation AND classification (machine learning) -# self._detectclassify_ObjectAll_PYWT() - - # general handling of all audio and video formats - self._detect_Streams_FFMPEG() - - # general file EXIF history information - self._detect_History_EXIF() - - # general audio feature extraction -# self._detect_AudioFeatures_YAAFE() - - # midi audio feature extraction -# TODO: improve midi/audio stuff ... - self._detect_AudioFeatures_MUSIC21() - def _existInformation(self, info, ignore = ['Properties', 'ColorAverage']): result = [] for item in info: @@ -3684,6 +3849,11 @@ result = self._info['Streams'] return {'Streams': result} +# def _filter_Audio(self): +# # use all, (should be reliable) +# result = self._info['Audio'] +# return {'Audio': result} + #def _filter_Geometry(self): # result = [] # for item in self._info['Geometry']:

1 0

SVN: [11620] branches/rewrite/pywikibot/userinterfaces
by valhallasw＠svn.wikimedia.org 06 Jun '13

06 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11620 Revision: 11620 Author: valhallasw Date: 2013-06-06 21:19:06 +0000 (Thu, 06 Jun 2013) Log Message: ----------- Removed disfunctional interfaces from rewrite Removed Paths: ------------- branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py Deleted: branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py =================================================================== --- branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py 2013-06-06 19:38:32 UTC (rev 11619) +++ branches/rewrite/pywikibot/userinterfaces/tkinter_interface.py 2013-06-06 21:19:06 UTC (rev 11620) @@ -1,229 +0,0 @@ -# -*- coding: utf-8 -*- -# -# (C) Pywikipedia bot team, 2005-2008 -# -# Distributed under the terms of the MIT license. -# -__version__ = '$Id$' - -import re -import sys -import threading -import time -import tkMessageBox, tkSimpleDialog -from Tkinter import * -from gui import EditBoxWindow - -color_pattern = re.compile(r"%s\{(?P<colorname>\w+)\}" % "\x03") - -# we run the Tkinter mainloop in a separate thread so as not to block -# the main bot code; however, this means that all communication with -# the Tkinter interface has to be done through events that will be processed -# by the mainloop in the separate thread. Code outside this module must not -# call any of the Tkinter objects directly. - - -class MainloopThread(threading.Thread): - def __init__(self, window): - threading.Thread.__init__(self) - self.window = window - - def run(self): - self.window.mainloop() - - -class CustomMessageBox(tkSimpleDialog.Dialog): - def __init__(self, master, question, options, hotkeys, default=None): - self.question = question - self.options = options - self.hotkeys = hotkeys - self.default = default - tkSimpleDialog.Dialog.__init__(self, master) - - def body(self, master): - Label(self, text=self.question).grid(columnspan = len(self.options)) - btns = [] - for i in xrange(len(self.options)): - # mark hotkey with underline - m = re.search('[%s%s]' % (self.hotkeys[i].lower(), - self.hotkeys[i].upper()), self.options[i]) - if m: - pos = m.start() - else: - self.options[i] += ' (%s)' % self.hotkeys[i] - pos = len(self.options[i]) - 2 - b = Button(self, text=self.options[i], - underline=pos, - command=lambda h=self.hotkeys[i]: self.select(h)) - self.bind("<Control-%s>" % self.hotkeys[i], - lambda h=self.hotkeys[i]: self.select(h)) - b.grid(row = 1, column = i) - btns.append(b) - if self.default and self.default in self.hotkeys: - return btns[self.hotkeys.index(self.default)] - else: - return btns[0] - - def buttonbox(self): - return - - def select(self, i, event=None): - self.selection = i - self.ok() - - def apply(self): - if self.default and not self.selection: - self.selection = self.default - - -class OutputBox(Text): - def __init__(self, parent, *args, **kwargs): - Text.__init__(self, parent, *args, **kwargs) - # create color tags - # map Unix/ANSI color names to Tcl/Tk names - # because Tkinter background is white, we need darker colors - for ucolor, tcolor in ( - ('default', 'Black'), - ('lightblue', 'Blue'), - ('lightgreen', 'Green'), - ('lightaqua', 'DarkSeaGreen'), - ('lightred', 'Red'), - ('lightpurple', 'DarkViolet'), - ('lightyellow', 'DarkOrange') - ): - self.tag_config(ucolor, foreground=tcolor) - - def show(self, text): - global debugger - debugger = text - next_tag = 'default' - m = color_pattern.search(text) - while m: - self.insert(END, text[0:m.start()], next_tag) - next_tag = m.group("colorname") - text = text[m.end():] - m = color_pattern.search(text) - self.insert(END, text, next_tag) - self.yview(END) - - -class TkController(Frame): - """ - Tkinter user interface controller. - - This object receives and processes events dispatched to it by the UI - object. Do not call this object's methods directly. Methods of this - object cannot return values to the main thread; they must store them - in attributes to be retrieved by the main thread. - - """ - # TODO: use Event for inter-thread communication instead of wait loops - def __init__(self, parent, **kwargs): - Frame.__init__(self, parent, **kwargs) - - def showinfo(self, text, wait=True): - """ - Show a pop-up message. - - Set wait to False to allow the pop-up to remain on screen while the - bot continues to work. - - """ - box = tkMessageBox.showinfo("Bot Message", text) - box.display() - if wait: - self.wait_window(box.top) - - def ask(self, question, password=False): - """ - Show a question in a dialog window and store the user's answer. - """ - self.answer = tkSimpleDialog.askstring('Question', question) - - def ask_choice(self, question, options, hotkeys, default): - d = CustomMessageBox(self, question, options, hotkeys) - self.wait_window(d.top) - self.selection = d.selection or d.default - - def edit_text(self, text, jumpIndex, highlight): - editBoxWindow = EditBoxWindow(text) - editBoxWindow.highlight(highlight) - self.wait_window(editBoxWindow.top) - self.edited_text = editBoxWindow.text - - -class UI(object): - """ - Tkinter user interface. - - This object serves only to dispatch event calls to the TkController - object, to be run in that object's separate mainloop thread; and, - when necessary, to wait for the user's response. - - """ - def __init__(self, parent = None): - # create a new window if necessary - self.control = TkController(parent or Tk()) - - # textarea with vertical scrollbar - scrollbar = Scrollbar(self.control) - self.logBox = OutputBox(self.control, yscrollcommand=scrollbar.set) - - # add scrollbar to main frame, associate it with our editbox - scrollbar.pack(side=RIGHT, fill=Y) - scrollbar.config(command=self.logBox.yview) - - # put textarea into top frame, using all available space - self.logBox.pack(anchor=CENTER, fill=BOTH) - self.control.pack(side=TOP) - - MainloopThread(self.control).start() - - def output(self, text, urgency=1, toStdout=False, wait=True): - """ - urgency levels: (NOT IMPLEMENTED) - 0 - Debug output. Won't be shown in normal mode. - 1 - Will be shown in log window. - 2 - Will be shown in error box. - - TODO: introduce constants - - wait: block bot until user dismisses pop-up window (level 2 only) - """ - if urgency >= 2: - self.control.after_idle(self.control.showinfo, text, wait) - elif urgency >= 1: - self.control.after_idle(self.logBox.show, text) - - def input(self, question, password = False): - """ - Returns a unicode string. - """ - # TODO: hide input if password = True - self.control.after_idle(self.control.ask, question) - # wait until the answer has been given - while not hasattr(self.control, "answer"): - time.sleep(1) - # answer needs to be deleted so that it won't be reused the - # next time this method is called - answer = self.control.answer - del self.control.answer - return answer - - def editText(self, text, jumpIndex = None, highlight = None): - self.control.after_idle(self.control.edit_text, - text, jumpIndex, highlight) - while not hasattr(self.control, "edited_text"): - time.sleep(1) - result = self.control.edited_text - del self.control.edited_text - return result - - def inputChoice(self, question, options, hotkeys, default = None): - self.control.after_idle(self.control.ask_choice, question, - options, hotkeys, default) - while not hasattr(self.control, "selection"): - time.sleep(1) - selection = self.control.selection - del self.control.selection - return selection Deleted: branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py =================================================================== --- branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py 2013-06-06 19:38:32 UTC (rev 11619) +++ branches/rewrite/pywikibot/userinterfaces/wxpython_interface.py 2013-06-06 21:19:06 UTC (rev 11620) @@ -1,64 +0,0 @@ - -__version__ = '$Id$' - -import sys; sys.path.append('..') - -import re -import terminal_interface -import wx - -app = wx.App() - -class UI(terminal_interface.UI): - def __init__(self): - pass - - def input(self, question, password = False): - """ - Works like raw_input(), but returns a unicode string instead of ASCII. - - Unlike raw_input, this function automatically adds a space after the - question. - """ - # TODO: hide input if password = True - - self.output(question) - if password: - answer = wx.PasswordEntryDialog( None, question, '','') - else: - answer = wx.TextEntryDialog( None, question, '', '' ) - answer.ShowModal() - self.output(answer+'\n') - #tkSimpleDialog.askstring('title', question) - return answer.GetValue() or '' - - def inputChoice(self, question, options, hotkeys, default = None): - for i in range(len(options)): - option = options[i] - hotkey = hotkeys[i] - m = re.search('[%s%s]' % (hotkey.lower(), hotkey.upper()), option) - if m: - pos = m.start() - options[i] = '%s[%s]%s' % (option[:pos], option[pos], option[pos+1:]) - else: - options[i] = '%s [%s]' % (option, hotkey) - - while True: - prompt = '%s\n(%s)' % (question, ', '.join(options)) - self.output('%s (%s)' % (question, ', '.join(options))) - answer = wx.TextEntryDialog(None, prompt, question, '') - answer.ShowModal() - answer = answer.GetValue() - self.output(answer+'\n') - - if answer.lower() in hotkeys or answer.upper() in hotkeys: - return answer - elif default and answer=='':# empty string entered - return default - -if __name__ == '__main__': - ui = UI() - print ui.input('Test?') - -app.MainLoop() -

1 0

SVN: [11619] branches/rewrite/pywikibot/site.py
by legoktm＠svn.wikimedia.org 06 Jun '13

06 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11619 Revision: 11619 Author: legoktm Date: 2013-06-06 19:38:32 +0000 (Thu, 06 Jun 2013) Log Message: ----------- Only add patrol flag if showPatrolled is not None, user can have either patrol or patrolmarks right. Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2013-06-02 17:01:35 UTC (rev 11618) +++ branches/rewrite/pywikibot/site.py 2013-06-06 19:38:32 UTC (rev 11619) @@ -2229,7 +2229,7 @@ 'anon': showAnon, 'redirect': showRedirects, } - if self.has_right('patrol'): + if showPatrolled is not None and (self.has_right('patrol') or self.has_right('patrolmarks')): rcgen.request['rcprop'] += '|patrolled' filters['patrolled'] = showPatrolled rcshow = []

1 0

SVN: [11618] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11618 Revision: 11618 Author: drtrigon Date: 2013-06-02 17:01:35 +0000 (Sun, 02 Jun 2013) Log Message: ----------- improvement; POSIT refactored into own method, does projection to calc errors also Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-02 13:07:05 UTC (rev 11617) +++ trunk/pywikipedia/catimages.py 2013-06-02 17:01:35 UTC (rev 11618) @@ -405,7 +405,7 @@ D2points = [ item[:2] + item[2:]/2. for item in D2points ] neutral = np.array([[np.pi],[0.],[0.]]) # calculate pose - rvec, tvec, cm, err = self._util_get_Pose(D3points, D2points, self.image_size) + rvec, tvec, cm, err = self._util_get_Pose_solvePnP(D3points, D2points, self.image_size) #data['Pose'] = tuple(rvec[:,0]) check = not (err[:,0,:].max() > 0.5) if not check: @@ -431,18 +431,18 @@ # drv, np.zeros((3,1)))[0] #print (rvec - nrv < 1E-12) # compare data['Pose'] = map(float, tuple(drv[:,0])) -# TODO: POSIT has to be tested and compared to solvePnP; which one is better?? +# TODO: POSIT has to be tested and compared; draw both results! # POSIT: http://www.cfar.umd.edu/~daniel/daniel_papersfordownload/Pose25Lines.pdf if False: pywikibot.output("solvePnP:") pywikibot.output(str(rvec[:,0])) pywikibot.output(str(tvec[:,0])) - import opencv - #opencv.unit_test() - (rmat, tvec, mdl) = opencv.posit(D3points, D2points, (100, 1.0e-4)) + pywikibot.output(str(err[:,0,:])) + rvec, tvec, cm, err = self._util_get_Pose_POSIT(D3points, D2points) pywikibot.output("POSIT:") - pywikibot.output(str(cv2.Rodrigues(rmat)[0][:,0])) + pywikibot.output(str(rvec[:,0])) pywikibot.output(str(tvec)) + pywikibot.output(str(np.array(err)[:,0,:]/max(self.image_size))) result.append( data ) ## see '_drawRect' @@ -455,7 +455,7 @@ self._info['Faces'] += result return - def _util_get_Pose(self, D3points, D2points, shape): + def _util_get_Pose_solvePnP(self, D3points, D2points, shape): """ Calculate pose from head model "little girl" w/o camera or other calibrations needed. @@ -488,6 +488,41 @@ return rvec, tvec, np.array(cameraMatrix), (np.array(err)/max_d) + #def _util_get_Pose_POSIT(self, D3points, D2points, shape): + def _util_get_Pose_POSIT(self, D3points, D2points): + """ Calculate pose from head model "little girl" w/o camera or other + calibrations needed. + + Method similar to '_util_get_Pose_solvePnP', please compare. + + D2points: left eye, right eye, nose, mouth + """ + # calculate pose + import opencv + #opencv.unit_test() + (rmat, tvec, mdl) = opencv.posit(D3points, D2points, (100, 1.0e-4)) + rvec = cv2.Rodrigues(rmat)[0] + + # Project the model points with the estimated pose + # http://opencv.willowgarage.com/documentation/cpp/camera_calibration_and_3d_… + # intrinsic: camera matrix + # extrinsic: rotation-translation matrix [R|t] + # CV_32F, principal point in the centre of the image is (0, 0) instead of (self.image_size[0]*0.5) + FOCAL_LENGTH = 760.0 # hard-coded in posit_python.cpp, should be changed... + cameraMatrix = [[FOCAL_LENGTH, 0.0, 0.0],#shape[0]*0.0], + [ 0.0, FOCAL_LENGTH, 0.0],#shape[1]*0.0], + [ 0.0, 0.0, 1.0],] + + # compare to 2D points + err = [] + for i, vec in enumerate(np.array(mdl)): + nvec = np.dot(cameraMatrix, (np.dot(rmat, vec) + tvec)) + err.append(((D2points[i] - nvec[:2]/nvec[2]), D2points[i], nvec[:2]/nvec[2])) + + #pywikibot.output(u'result for UN-calibrated camera:\n rot=%s' % rvec.transpose()[0]) + + return rvec, tvec, np.array(cameraMatrix), (np.array(err)/1.0) + # https://pypi.python.org/pypi/xbob.flandmark # http://cmp.felk.cvut.cz/~uricamic/flandmark/ def _detect_FaceLandmark_xBOB(self): @@ -3849,9 +3884,9 @@ #outresult = [ outpage.get() ] + outresult # append to page outresult = u"\n".join(outresult) pywikibot.output(u"Size of log page data: %s byte(s)" % len(outresult)) - # write pages mutliple times if content is too large in order to circumvent: + # work-a-round: write pages mutliple times if content is too large in order to circumvent # "HTTPError: 504 Gateway Time-out" leading finally to "MaxTriesExceededError" - # (why is that...?!?? FIX THIS!) + # (why is that...?!?? FIX THIS in the framework core e.g. 'postForm'!) tmp = outresult while tmp: i = np.array([m.start() for m in re.finditer(u"\n\n==", tmp)]

1 0

SVN: [11617] branches/rewrite/tests/page_tests.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11617 Revision: 11617 Author: legoktm Date: 2013-06-02 13:07:05 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Add tests for page.get() Modified Paths: -------------- branches/rewrite/tests/page_tests.py Modified: branches/rewrite/tests/page_tests.py =================================================================== --- branches/rewrite/tests/page_tests.py 2013-06-02 12:49:02 UTC (rev 11616) +++ branches/rewrite/tests/page_tests.py 2013-06-02 13:07:05 UTC (rev 11617) @@ -215,6 +215,16 @@ self.assertTrue(p1.isRedirectPage()) self.assertEqual(p1.getRedirectTarget(), p2) + def testPageGet(self): + p1 = pywikibot.Page(site, u'User:Legoktm/R2') + p2 = pywikibot.Page(site, u'User:Legoktm/R1') + p3 = pywikibot.Page(site, u'User:Legoktm/R3') + + text = u'This page is used in the [[mw:Manual:Pywikipediabot]] testing suite.' + self.assertEqual(p1.get(), text) + self.assertRaises(pywikibot.exceptions.IsRedirectPage, p2.get) + self.assertRaises(pywikibot.exceptions.NoPage, p3.get) + def testApiMethods(self): """Test various methods that rely on API.""" # since there is no way to predict what data the wiki will return, @@ -307,7 +317,6 @@ ## def autoFormat(self): ## def isAutoTitle(self): -## def get(self, force=False, get_redirect=False, sysop=False): ## def getOldVersion(self, oldid, force=False, get_redirect=False, ## sysop=False): ## text = property(_textgetter, _textsetter, _cleartext,

1 0

SVN: [11616] branches/rewrite/tests/page_tests.py
by legoktm＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11616 Revision: 11616 Author: legoktm Date: 2013-06-02 12:49:02 +0000 (Sun, 02 Jun 2013) Log Message: ----------- Add test for page.getRedirectTarget() Modified Paths: -------------- branches/rewrite/tests/page_tests.py Modified: branches/rewrite/tests/page_tests.py =================================================================== --- branches/rewrite/tests/page_tests.py 2013-06-02 12:15:06 UTC (rev 11615) +++ branches/rewrite/tests/page_tests.py 2013-06-02 12:49:02 UTC (rev 11616) @@ -209,6 +209,12 @@ self.assertEqual(p2.isImage(), True) self.assertEqual(p3.isImage(), False) + def testIsRedirect(self): + p1 = pywikibot.Page(site, u'User:Legoktm/R1') + p2 = pywikibot.Page(site, u'User:Legoktm/R2') + self.assertTrue(p1.isRedirectPage()) + self.assertEqual(p1.getRedirectTarget(), p2) + def testApiMethods(self): """Test various methods that rely on API.""" # since there is no way to predict what data the wiki will return, @@ -312,7 +318,6 @@ ## def backlinks(self, followRedirects=True, filterRedirects=None, ## namespaces=None): ## def embeddedin(self, filter_redirects=None, namespaces=None): -## def getRedirectTarget(self): ## def getVersionHistory(self, reverseOrder=False, getAll=False, ## revCount=500): ## def getVersionHistoryTable(self, forceReload=False, reverseOrder=False,

1 0

SVN: [11615] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 02 Jun '13

02 Jun '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11615 Revision: 11615 Author: drtrigon Date: 2013-06-02 12:15:06 +0000 (Sun, 02 Jun 2013) Log Message: ----------- bug fix; work-a-round for MaxTriesExceededError(s) caused due to to big page content write attempt (should be solved properly in framework core, e.g. 'postData') Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-06-02 12:06:55 UTC (rev 11614) +++ trunk/pywikipedia/catimages.py 2013-06-02 12:15:06 UTC (rev 11615) @@ -432,6 +432,7 @@ #print (rvec - nrv < 1E-12) # compare data['Pose'] = map(float, tuple(drv[:,0])) # TODO: POSIT has to be tested and compared to solvePnP; which one is better?? + # POSIT: http://www.cfar.umd.edu/~daniel/daniel_papersfordownload/Pose25Lines.pdf if False: pywikibot.output("solvePnP:") pywikibot.output(str(rvec[:,0])) @@ -3474,6 +3475,7 @@ self._detect_Faces_EXIF() # Faces and eyes (opencv pre-trained haar) self._detect_Faces_CV() +# TODO: test and use or switch off # Face via Landmark(s) # self._detect_FaceLandmark_xBOB() # exclude duplicates (CV and EXIF) @@ -3507,7 +3509,6 @@ self._recognize_OpticalCodes_dmtxNzbar() # Chessboard (opencv reference detector) -# TODO: apply pose estimation/detection to faces (eyes, mouth, ... or landmark) also self._detect_Chessboard_CV() # general (self-trained) detection WITH classification @@ -3846,13 +3847,26 @@ if outresult: outpage = pywikibot.Page(site, u"User:DrTrigon/User:DrTrigonBot/logging") #outresult = [ outpage.get() ] + outresult # append to page - outpage.put( u"\n".join(outresult), comment="bot writing log for last run" ) + outresult = u"\n".join(outresult) + pywikibot.output(u"Size of log page data: %s byte(s)" % len(outresult)) + # write pages mutliple times if content is too large in order to circumvent: + # "HTTPError: 504 Gateway Time-out" leading finally to "MaxTriesExceededError" + # (why is that...?!?? FIX THIS!) + tmp = outresult + while tmp: + i = np.array([m.start() for m in re.finditer(u"\n\n==", tmp)] + + [len(tmp)]) + #pos = i[ np.where((i - 2048*1024) <= 0)[0][-1] ] # $wgMaxArticleSize + pos = i[ np.where((i - 500*1024) <= 0)[0][-1] ] + pywikibot.output(u"Size of bunch to write: %s byte(s)" % len(tmp[:pos])) + outpage.put( tmp[:pos], comment="bot writing log for last run" ) + tmp = tmp[pos:] if pywikibot.simulate: #print u"--- " * 20 #print u"--- " * 20 - #print u"\n".join(outresult) + #print outresult posfile = open(os.path.join(scriptdir, 'cache/catimages.log'), "a") - posfile.write( u"\n".join(outresult) ) + posfile.write( outresult ) posfile.close() # http://scipy-lectures.github.com/advanced/scikit-learn/index.html

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn June 2013