[Pywikibot-commits] [Gerrit] Do not force preloading patch.exe for windows user. Ask for ... - change (pywikibot/compat)

Xqt has submitted this change and it was merged. Change subject: Do not force preloading patch.exe for windows user. Ask for it if needed only and prohibit this message every time a script was started. Otherwise automatic batch scripts could not run without operator inputs. Some additional PEP8 changes and small improv ...................................................................... Do not force preloading patch.exe for windows user. Ask for it if needed only and prohibit this message every time a script was started. Otherwise automatic batch scripts could not run without operator inputs. Some additional PEP8 changes and small improvements. Change-Id: Icc887384f2d8dea1a03c17213a212c7f5bbb3533 --- M externals/__init__.py 1 file changed, 106 insertions(+), 98 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/externals/__init__.py b/externals/__init__.py index db6baf5..314c55d 100644 --- a/externals/__init__.py +++ b/externals/__init__.py @@ -31,26 +31,26 @@ # mercurial (hg) # patch (unix/linux & gnuwin32 version/flavour) modules_needed = { - 'patch.exe': ({}, # for win32 only, unix/linux is already equipped with a patch tool + 'patch.exe': ({}, # for win32 only, unix/linux is already equipped with a patch tool { 'url': 'http://downloads.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip', 'path': 'bin/patch.exe'}, - {}), # OK + {}), # OK 'crontab': ({}, - #{ 'url': 'https://github.com/josiahcarlson/parse-crontab/archive/master.zip', - # 'path': 'parse-crontab-master/crontab',}), # OK +## { 'url': 'https://github.com/josiahcarlson/parse-crontab/archive/master.zip', +## 'path': 'parse-crontab-master/crontab',}), # OK { 'url': 'https://github.com/josiahcarlson/parse-crontab/archive/1ec538ff67df6a207993a6c5b6988f4f628c5776.zip', 'path': 'parse-crontab-1ec538ff67df6a207993a6c5b6988f4f628c5776/crontab',}, - {}), # OK + {}), # OK 'odf': ({}, - #{ 'url': 'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.6.tar.gz', - # 'path': 'odfpy-0.9.6/odf',}), # OK +## { 'url': 'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.6.tar.gz', +## 'path': 'odfpy-0.9.6/odf',}), # OK { 'url': 'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.4.tar.gz', 'path': 'odfpy-0.9.4/odf'}, - {}), # OK + {}), # OK 'openpyxl': ({}, { 'url': 'https://bitbucket.org/ericgazoni/openpyxl/get/1.5.6.tar.gz', 'path': 'ericgazoni-openpyxl-e5934500ffac/openpyxl'}, - {}), # OK + {}), # OK # 'spelling': $ svn propedit svn:externals externals/. # spelling http://svn.wikimedia.org/svnroot/pywikipedia/trunk/spelling/ # $ git submodule add https://gerrit.wikimedia.org/r/p/pywikibot/spelling.git externals/spelling @@ -58,46 +58,46 @@ 'linux-ubuntu': ['python-beautifulsoup']}, { 'url': 'https://pypi.python.org/packages/source/B/BeautifulSoup/BeautifulSoup-3.2.0.tar.gz', 'path': 'BeautifulSoup-3.2.0/BeautifulSoup.py'}, - {}), # OK + {}), # OK 'irclib': ({'linux-fedora': ['python-irclib'], 'linux-ubuntu': ['python-irclib']}, - {}, # http://python-irclib.sourceforge.net/ - {}), # OK + {}, # http://python-irclib.sourceforge.net/ + {}), # OK 'mwparserfromhell': ({}, { 'url': 'https://github.com/earwig/mwparserfromhell/archive/v0.2.zip', - #{ 'url': 'https://github.com/earwig/mwparserfromhell/archive/master.zip', +## { 'url': 'https://github.com/earwig/mwparserfromhell/archive/master.zip', 'path': 'mwparserfromhell-0.2/mwparserfromhell'}, - {}), # OK + {}), # OK 'colormath': ({'linux-fedora': [], 'linux-ubuntu': ['python-colormath'],}, { 'url': 'https://github.com/gtaylor/python-colormath/archive/master.zip', 'path': 'python-colormath-master/colormath',}, - {}), # OK + {}), # OK 'jseg': ({}, { 'url': 'http://vision.ece.ucsb.edu/segmentation/jseg/software/jseg.zip', 'path': 'jseg', #$ diff -Nau --exclude="*.o" --exclude="*.pyc" --exclude="segdist_cpp*" TEST_jseg/ jseg/ > patch-jseg 'patch': 'patch-jseg'}, - {}), # OK + {}), # OK 'jseg/jpeg-6b': ({}, { 'url': 'http://vision.ece.ucsb.edu/segmentation/jseg/software/jpeg-6b.zip', 'path': 'jpeg-6b',}, - {}), # OK + {}), # OK '_mlpy': ({}, { 'url': 'http://downloads.sourceforge.net/project/mlpy/mlpy%203.5.0/mlpy-3.5.0.tar.gz', 'path': 'mlpy-3.5.0/mlpy'}, - {}), # OK + {}), # OK '_music21': ({}, { 'url': 'http://music21.googlecode.com/files/music21-1.4.0.tar.gz', 'path': 'music21-1.4.0', #$ diff -Naur --exclude="*.pyc" TEST__music21/ _music21/ > patch-music21 'patch': 'patch-music21'}, - {}), # OK + {}), # OK # TODO: vvv (future; enable for and use in 'catimages.py', patch needed) '_ocropus': ({}, {}, { 'url': 'https://code.google.com/p/ocropus', - 'rev': 'ocropus-0.6'}), # OK + 'rev': 'ocropus-0.6'}), # OK # TODO: vvv (further clean-up and unlink - check with 'svn list') # 'opencv': $ svn propedit svn:externals externals/. # opencv https://svn.toolserver.org/svnroot/drtrigon/externals/opencv @@ -111,7 +111,7 @@ 'opencv/haarcascades': ({}, { 'url': 'https://svn.toolserver.org/svnroot/drtrigon/externals/haarcascades-full.tar.gz', 'path': 'haarcascades'}, - {}), # OK + {}), # OK # 'pdfminer' is not used anymore/at the moment... # 'pycolorname': $ svn propset svn:externals 'pycolorname https://svn.toolserver.org/svnroot/drtrigon/externals/pycolorname' externals/. 'pydmtx': ({'linux-fedora': ['python-libdmtx'], @@ -120,11 +120,11 @@ 'path': 'dmtx-wrappers-master/python', #$ diff -Nau --exclude="*.pyc" TEST_pydmtx/ pydmtx/ > patch-pydmtx 'patch': 'patch-pydmtx'}, - {}), # OK + {}), # OK 'py_w3c': ({}, { 'url': 'https://bitbucket.org/nmb10/py_w3c/downloads/py_w3c-v0.1.0.tar.gz', 'path': 'py_w3c-0.1.0/py_w3c'}, - {}), # OK + {}), # OK # TODO: vvv (include) # 'TEST_slic': ({}, # { 'url': 'http://ivrg.epfl.ch/files/content/sites/ivrg/files/supplementary_material/RK_SLICsuperpixels/SLICSuperpixelsAndSupervoxelsCode.zip', @@ -139,7 +139,7 @@ 'path': 'zbar-0.10', #$ diff -Nau --exclude="*.pyc" TEST__zbar/ _zbar/ > patch-zbar 'patch': 'patch-zbar'}, - {}), # OK + {}), # OK # TODO: vvv (include) # 'TEST__bob': ({}, # { 'url': 'https://www.idiap.ch/software/bob/packages/bob-1.1.2.zip', @@ -168,14 +168,14 @@ import os import sys import inspect -import wikipedia as pywikibot # sets externals path +import wikipedia as pywikibot # sets externals path #from pywikibot.comms import http # allow imports from externals sys.path.append(os.path.dirname(os.path.abspath(__file__))) -### BEGIN of VisTrails inspired and copied code ### ### ### ### ### ### ### ### +### BEGIN of VisTrails inspired and copied code ### def has_logger(): #return hasattr(sys.modules['wikipedia'], 'logger') @@ -190,7 +190,7 @@ if has_logger(): pywikibot.warning(text) else: - print "WARNING:", text + print "\nWARNING:", text def guess_system(): @@ -201,19 +201,20 @@ def show_question(module): lowlevel_warning("Required package missing: %s\n" "This package is not installed, but required by the file" - " '%s'." % (module, inspect.stack()[2][1])) + "\n'%s'." % (module, inspect.stack()[2][1])) lowlevel_warning("For more and additional information, please confer:\n" "http://www.mediawiki.org/wiki/Manual:Pywikipediabot/" "Installation#Dependencies") options = [(i+1) for i, item in enumerate(modules_needed[module]) if item] options += [0, 's', ''] options.sort() - options_msg = ("There are multiple ways to solve this:\n" - "RECOMMENDED for admins: always option [0] or the next available" - " (e.g. [1])\n" - "RECOMMENDED for non-admins: always option [2] (if available)\n" - "0: automatically determine the best of the following methods (may need\n" - " administrator privileges)\n") + options_msg = ("""There are multiple ways to solve this: +RECOMMENDED for admins: always option [0] or the next available (e.g. [1]) +RECOMMENDED for non-admins: always option [2] (if available) +0: automatically determine the best of the following methods (may need + administrator privileges) + +""") if 1 in options: options_msg += ("1: install the package using the OS package" " management system like yum\n" @@ -230,8 +231,8 @@ lowlevel_warning(options_msg) v = None while (v not in options): - lowlevel_warning("Please choose [%s, s - default]: " % - (", ".join(map(str, options[:-2])))) + lowlevel_warning("Please choose [%s, s - default]: " + % (", ".join(map(str, options[:-2])))) v = raw_input().lower() try: v = int(v) @@ -239,11 +240,13 @@ pass return v + def show_patch_question(): global _patch_permission if _patch_permission is None: - lowlevel_warning("Give externals permission to execute the patch command?" - " [y(es), n(o) - default]: ") + lowlevel_warning( + "Give externals permission to execute the patch command?" + " [y(es), n(o) - default]: ") v = raw_input().upper() _patch_permission = (v == 'Y') or (v == 'YES') return _patch_permission @@ -251,7 +254,9 @@ def python_module_exists(module_name): """python_module_exists(module_name): Boolean. -Returns if python module of given name can be safely imported.""" + Returns if python module of given name can be safely imported. + + """ module_name = module_name.replace(u'.py', u'') module_name = module_name[1:] if module_name[0] == u'_' else module_name @@ -301,9 +306,11 @@ def linux_install(dependency_dictionary): - """Tries to import a python module. If unsuccessful, tries to install -the appropriate bundle and then reimport. py_import tries to be smart -about which system it runs on.""" + """ Tries to import a python module. If unsuccessful, tries to install + the appropriate bundle and then reimport. py_import tries to be smart + about which system it runs on. + + """ # Ugly fix to avoid circular import distro = guess_system() @@ -330,54 +337,60 @@ def windows_install(dependency_dictionary): - lowlevel_warning(u'Not available in windows OS, use download mode (2) instead.') + lowlevel_warning( + u'Not available in windows OS, use download mode (2) instead.') return False # skip this in order to trigger 'download_install' next ### END of VisTrails inspired and copied code ### ### ### ### ### ### ### ### def download_install(package, module, path): - if package: - lowlevel_warning(u'Download package "%s" from %s' - % (module, package['url'])) - import mimetypes - import urllib2 - for i in range(3): - response = urllib2.urlopen(package['url']) - #response = http.request(pywikibot.getSite(), package['url'], - # no_hostname = True, back_response = True)[0] - if 'Content-Length' in response.headers: - break - lowlevel_warning(u'Could not retrieve data, re-trying ...') - lowlevel_warning(u'Size of download: %s byte(s)' - % response.headers['Content-Length']) - #mime = response.headers['Content-Type'].lower().split('/') - mime = mimetypes.guess_type(package['url'], - strict=True)[0].lower().split('/') - lowlevel_warning(u'MIME type: %s' % mime) + if not package: + return + lowlevel_warning(u'Download package "%s" from %s' + % (module, package['url'])) + import mimetypes + import urllib2 + for i in range(3): + response = urllib2.urlopen(package['url']) + #response = http.request(pywikibot.getSite(), package['url'], + # no_hostname = True, back_response = True)[0] + if 'Content-Length' in response.headers: + break + lowlevel_warning(u'Could not retrieve data, re-trying ...') + lowlevel_warning(u'Size of download: %s byte(s)' + % response.headers['Content-Length']) + #mime = response.headers['Content-Type'].lower().split('/') + mime = mimetypes.guess_type(package['url'], + strict=True)[0].lower().split('/') + lowlevel_warning(u'MIME type: %s' % mime) - lowlevel_warning(u'Extract package "%s" to %s.' - % (module, os.path.join(path, module))) - if len(mime) > 1: - import StringIO - if mime[1] in ['zip', 'x-zip-compressed']: - import zipfile - arch = zipfile.ZipFile(StringIO.StringIO(response.read())) - elif mime[1] == 'x-tar': - import tarfile - arch = tarfile.open(fileobj=StringIO.StringIO(response.read())) - else: - raise NotImplementedError(u'Not implemented mime type %s' - % mime[1]) - arch.extractall(os.path.join(path, '__setup_tmp/')) - arch.close() - import shutil - shutil.move(os.path.join(path, '__setup_tmp/', package['path']), - os.path.join(path, module)) - shutil.rmtree(os.path.join(path, '__setup_tmp/')) + lowlevel_warning(u'Extract package "%s" to %s.' + % (module, os.path.join(path, module))) + if len(mime) > 1: + import StringIO + if mime[1] in ['zip', 'x-zip-compressed']: + import zipfile + arch = zipfile.ZipFile(StringIO.StringIO(response.read())) + elif mime[1] == 'x-tar': + import tarfile + arch = tarfile.open(fileobj=StringIO.StringIO(response.read())) + else: + raise NotImplementedError(u'Not implemented mime type %s' + % mime[1]) + arch.extractall(os.path.join(path, '__setup_tmp/')) + arch.close() + import shutil + shutil.move(os.path.join(path, '__setup_tmp/', package['path']), + os.path.join(path, module)) + shutil.rmtree(os.path.join(path, '__setup_tmp/')) - result = 0 - if ('patch' in package) and show_patch_question(): + result = 0 + if 'patch' in package: + if sys.platform == 'win32': + if not check_setup('patch.exe'): + result = -1 + if result == 0 and show_patch_question(): lowlevel_warning(u'Applying patch to %s in order to finish' u' the installation of package "%s".' % (os.path.join(path, module), module)) @@ -391,9 +404,9 @@ os.path.join(path, package['patch'])) result = os.system(cmd) - lowlevel_warning(u'Package "%s" installed to %s.' - % (module, os.path.join(path, module))) - return (result == 0) + lowlevel_warning(u'Package "%s" installed to %s.' + % (module, os.path.join(path, module))) + return (result == 0) def mercurial_repo_install(package, module, path): @@ -423,19 +436,21 @@ dist = guess_system() func = dist.split(u'-')[0] + '_install' if sel in [0, 1]: - lowlevel_warning(u'(1) Trying to install by use of "%s" package management system:' % dist) + lowlevel_warning( + u'(1) Trying to install by use of "%s" package management system:' + % dist) if (func in globals()) and globals()[func](modules_needed[m][0]): - return + return sel if sel in [0, 2]: lowlevel_warning(u'(2) Trying to install by download from source URL:') if download_install(modules_needed[m][1], m, path): - return + return sel if sel in [0, 3]: lowlevel_warning(u'(3) Trying to install by use of mercurial:') - if (len(modules_needed[m]) > 2) and\ + if (len(modules_needed[m]) > 2) and \ mercurial_repo_install(modules_needed[m][2], m, path): - return - if sel in [0, 1, 2, 3]: + return sel + if sel in range(4): lowlevel_warning(u'No suitable package could be found nor installed!') lowlevel_warning(u'Several scripts might fail, if the modules are not' @@ -448,10 +463,3 @@ def check_setup_all(): for m in modules_order: check_setup(m) - - -# check and install modules NEEDED -if sys.platform == 'win32': - check_setup('patch.exe') -#check_setup('BeautifulSoup.py') -#check_setup('mwparserfromhell') -- To view, visit https://gerrit.wikimedia.org/r/83114 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icc887384f2d8dea1a03c17213a212c7f5bbb3533 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot