Package mvpa :: Package base :: Module externals
[hide private]
[frames] | no frames]

Source Code for Module mvpa.base.externals

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Helper to verify presence of external libraries and modules 
 10  """ 
 11   
 12  __docformat__ = 'restructuredtext' 
 13  import os 
 14   
 15  from mvpa.base import warning 
 16  from mvpa import cfg 
 17  from mvpa.misc.support import SmartVersion 
 18   
 19  if __debug__: 
 20      from mvpa.base import debug 
 21   
22 -class _VersionsChecker(dict):
23 """Helper class to check the versions of the available externals 24 """
25 - def __getitem__(self, key):
26 if not self.has_key(key): 27 exists(key, force=True, raiseException=True) 28 return super(_VersionsChecker, self).__getitem__(key)
29 30 versions = _VersionsChecker() 31 """Versions of available externals, as tuples 32 """ 33 34
35 -def __check_scipy():
36 """Check if scipy is present an if it is -- store its version 37 """ 38 import warnings 39 exists('numpy', raiseException=True) 40 # To don't allow any crappy warning to sneak in 41 warnings.simplefilter('ignore', DeprecationWarning) 42 try: 43 import scipy as sp 44 except: 45 warnings.simplefilter('default', DeprecationWarning) 46 raise 47 warnings.simplefilter('default', DeprecationWarning) 48 # Infiltrate warnings if necessary 49 numpy_ver = versions['numpy'] 50 scipy_ver = versions['scipy'] = SmartVersion(sp.__version__) 51 # There is way too much deprecation warnings spit out onto the 52 # user. Lets assume that they should be fixed by scipy 0.7.0 time 53 if scipy_ver >= "0.6.0" and scipy_ver < "0.7.0" \ 54 and numpy_ver > "1.1.0": 55 import warnings 56 if not __debug__ or (__debug__ and not 'PY' in debug.active): 57 debug('EXT', "Setting up filters for numpy DeprecationWarnings") 58 filter_lines = [ 59 ('NumpyTest will be removed in the next release.*', 60 DeprecationWarning), 61 ('PyArray_FromDims: use PyArray_SimpleNew.', 62 DeprecationWarning), 63 ('PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr.', 64 DeprecationWarning), 65 # Trick re.match, since in warnings absent re.DOTALL in re.compile 66 ('[\na-z \t0-9]*The original semantics of histogram is scheduled to be.*' 67 '[\na-z \t0-9]*', Warning) ] 68 for f, w in filter_lines: 69 warnings.filterwarnings('ignore', f, w)
70 71
72 -def __check_numpy():
73 """Check if numpy is present (it must be) an if it is -- store its version 74 """ 75 import numpy as N 76 versions['numpy'] = SmartVersion(N.__version__)
77 78
79 -def __check_pywt(features=None):
80 """Check for available functionality within pywt 81 82 :Parameters: 83 features : list of basestring 84 List of known features to check such as 'wp reconstruct', 85 'wp reconstruct fixed' 86 """ 87 import pywt 88 import numpy as N 89 data = N.array([ 0.57316901, 0.65292526, 0.75266733, 0.67020084, 0.46505364, 90 0.76478331, 0.33034164, 0.49165547, 0.32979941, 0.09696717, 91 0.72552711, 0.4138999 , 0.54460628, 0.786351 , 0.50096306, 92 0.72436454, 0.2193098 , -0.0135051 , 0.34283984, 0.65596245, 93 0.49598417, 0.39935064, 0.26370727, 0.05572373, 0.40194438, 94 0.47004551, 0.60327258, 0.25628266, 0.32964893, 0.24009889,]) 95 mode = 'per' 96 wp = pywt.WaveletPacket(data, 'sym2', mode) 97 wp2 = pywt.WaveletPacket(data=None, wavelet='sym2', mode=mode) 98 try: 99 for node in wp.get_level(2): wp2[node.path] = node.data 100 except: 101 raise ImportError, \ 102 "Failed to reconstruct WP by specifying data in the layer" 103 104 if 'wp reconstruct fixed' in features: 105 rec = wp2.reconstruct() 106 if N.linalg.norm(rec[:len(data)] - data) > 1e-3: 107 raise ImportError, \ 108 "Failed to reconstruct WP correctly" 109 return True
110 111
112 -def __check_libsvm_verbosity_control():
113 """Check for available verbose control functionality 114 """ 115 import mvpa.clfs.libsvmc._svmc as _svmc 116 try: 117 _svmc.svm_set_verbosity(0) 118 except: 119 raise ImportError, "Provided version of libsvm has no way to control " \ 120 "its level of verbosity"
121
122 -def __check_shogun(bottom_version, custom_versions=[]):
123 """Check if version of shogun is high enough (or custom known) to 124 be enabled in the testsuite 125 126 :Parameters: 127 bottom_version : int 128 Bottom version which must be satisfied 129 custom_versions : list of int 130 Arbitrary list of versions which could got patched for 131 a specific issue 132 """ 133 import shogun.Classifier as __sc 134 ver = __sc.Version_get_version_revision() 135 if (ver in custom_versions) or (ver >= bottom_version): 136 return True 137 else: 138 raise ImportError, 'Version %s is smaller than needed %s' % \ 139 (ver, bottom_version)
140 141
142 -def __check_weave():
143 """Apparently presence of scipy is not sufficient since some 144 versions experience problems. E.g. in Sep,Oct 2008 lenny's weave 145 failed to work. May be some other converter could work (? See 146 http://lists.debian.org/debian-devel/2008/08/msg00730.html for a 147 similar report. 148 149 Following simple snippet checks compilation of the basic code using 150 weave 151 """ 152 from scipy import weave 153 from scipy.weave import converters, build_tools 154 import numpy as N 155 # to shut weave up 156 import sys 157 # we can't rely on weave at all at the restoring argv. On etch box 158 # restore_sys_argv() is apparently is insufficient 159 oargv = sys.argv[:] 160 ostdout = sys.stdout 161 if not( __debug__ and 'EXT_' in debug.active): 162 from StringIO import StringIO 163 sys.stdout = StringIO() 164 # *nix specific solution to shut weave up. 165 # Some users must complain and someone 166 # needs to fix this to become more generic. 167 cargs = [">/dev/null", "2>&1"] 168 else: 169 cargs = [] 170 fmsg = None 171 try: 172 data = N.array([1,2,3]) 173 counter = weave.inline("data[0]=fabs(-1);", ['data'], 174 type_converters=converters.blitz, 175 verbose=0, 176 extra_compile_args=cargs, 177 compiler = 'gcc') 178 except Exception, e: 179 fmsg = "Failed to build simple weave sample." \ 180 " Exception was %s" % str(e) 181 182 sys.stdout = ostdout 183 # needed to fix sweave which might "forget" to restore sysv 184 # build_tools.restore_sys_argv() 185 sys.argv = oargv 186 if fmsg is not None: 187 raise ImportError, fmsg 188 else: 189 return "Everything is cool"
190 191
192 -def __check_atlas_family(family):
193 # XXX I guess pylint will dislike it a lot 194 from mvpa.atlases.warehouse import KNOWN_ATLAS_FAMILIES 195 names, pathpattern = KNOWN_ATLAS_FAMILIES[family] 196 filename = pathpattern % {'name':names[0]} 197 if not os.path.exists(filename): 198 raise ImportError, "Cannot find file %s for atlas family %s" \ 199 % (filename, family) 200 pass
201 202
203 -def __check_stablerdist():
204 import scipy.stats 205 import numpy as N 206 ## Unfortunately 0.7.0 hasn't fixed the issue so no chance but to do 207 ## a proper numerical test here 208 try: 209 scipy.stats.rdist(1.32, 0, 1).cdf(-1.0 + N.finfo(float).eps) 210 # Actually previous test is insufficient for 0.6, so enabling 211 # elderly test on top 212 # ATM all known implementations which implement custom cdf for 213 # rdist are misbehaving, so there should be no _cdf 214 if '_cdf' in scipy.stats.distributions.rdist_gen.__dict__.keys(): 215 raise ImportError, \ 216 "scipy.stats carries misbehaving rdist distribution" 217 except ZeroDivisionError: 218 raise RuntimeError, "RDist in scipy is still unstable on the boundaries"
219 220
221 -def __check_rv_discrete_ppf():
222 """Unfortunately 0.6.0-12 of scipy pukes on simple ppf 223 """ 224 import scipy.stats 225 try: 226 bdist = scipy.stats.binom(100, 0.5) 227 bdist.ppf(0.9) 228 except TypeError: 229 raise RuntimeError, "pmf is broken in discrete dists of scipy.stats"
230 231
232 -def __check_in_ipython():
233 # figure out if ran within IPython 234 if '__IPYTHON__' in globals()['__builtins__']: 235 return 236 raise RuntimeError, "Not running in IPython session"
237 238
239 -def __check_matplotlib():
240 """Check for presence of matplotlib and set backend if requested.""" 241 import matplotlib 242 backend = cfg.get('matplotlib', 'backend') 243 if backend: 244 matplotlib.use(backend)
245
246 -def __check_pylab():
247 """Check if matplotlib is there and then pylab""" 248 exists('matplotlib', raiseException=True) 249 import pylab as P
250
251 -def __check_pylab_plottable():
252 """Simple check either we can plot anything using pylab. 253 254 Primary use in unittests 255 """ 256 try: 257 exists('pylab', raiseException=True) 258 import pylab as P 259 fig = P.figure() 260 P.plot([1,2], [1,2]) 261 P.close(fig) 262 except: 263 raise RuntimeError, "Cannot plot in pylab" 264 return True
265 266
267 -def __check_griddata():
268 """griddata might be independent module or part of mlab 269 """ 270 271 try: 272 from griddata import griddata as __ 273 return True 274 except ImportError: 275 if __debug__: 276 debug('EXT_', 'No python-griddata available') 277 278 from matplotlib.mlab import griddata as __ 279 return True
280 281
282 -def __check_reportlab():
283 import reportlab as rl 284 versions['reportlab'] = SmartVersion(rl.Version)
285 286
287 -def __check_rpy():
288 """Check either rpy is available and also set it for the sane execution 289 """ 290 #import rpy_options 291 #rpy_options.set_options(VERBOSE=False, SETUP_READ_CONSOLE=False) # SETUP_WRITE_CONSOLE=False) 292 #rpy_options.set_options(VERBOSE=False, SETUP_WRITE_CONSOLE=False) # SETUP_WRITE_CONSOLE=False) 293 # if not cfg.get('rpy', 'read_console', default=False): 294 # print "no read" 295 # rpy_options.set_options(SETUP_READ_CONSOLE=False) 296 # if not cfg.get('rpy', 'write_console', default=False): 297 # print "no write" 298 # rpy_options.set_options(SETUP_WRITE_CONSOLE=False) 299 import rpy 300 if not cfg.getboolean('rpy', 'interactive', default=True) \ 301 and (rpy.get_rpy_input() is rpy.rpy_io.rpy_input): 302 if __debug__: 303 debug('EXT_', "RPy: providing dummy callback for input to return '1'") 304 def input1(*args): return "1" # which is "1: abort (with core dump, if enabled)" 305 rpy.set_rpy_input(input1)
306 307 308 # contains list of available (optional) external classifier extensions 309 _KNOWN = {'libsvm':'import mvpa.clfs.libsvmc._svm as __; x=__.convert2SVMNode', 310 'libsvm verbosity control':'__check_libsvm_verbosity_control();', 311 'nifti':'from nifti import NiftiImage as __', 312 'nifti ge 0.20090205.1': 313 'from nifti.clib import detachDataFromImage as __', 314 'ctypes':'import ctypes as __', 315 'shogun':'import shogun as __', 316 'shogun.mpd': 'import shogun.Classifier as __; x=__.MPDSVM', 317 'shogun.lightsvm': 'import shogun.Classifier as __; x=__.SVMLight', 318 'shogun.svrlight': 'from shogun.Regression import SVRLight as __', 319 'numpy': "__check_numpy()", 320 'scipy': "__check_scipy()", 321 'good scipy.stats.rdist': "__check_stablerdist()", 322 'good scipy.stats.rv_discrete.ppf': "__check_rv_discrete_ppf()", 323 'weave': "__check_weave()", 324 'pywt': "import pywt as __", 325 'pywt wp reconstruct': "__check_pywt(['wp reconstruct'])", 326 'pywt wp reconstruct fixed': "__check_pywt(['wp reconstruct fixed'])", 327 'rpy': "__check_rpy()", 328 'lars': "exists('rpy', raiseException=True); import rpy; rpy.r.library('lars')", 329 'elasticnet': "exists('rpy', raiseException=True); import rpy; rpy.r.library('elasticnet')", 330 'glmnet': "exists('rpy', raiseException=True); import rpy; rpy.r.library('glmnet')", 331 'matplotlib': "__check_matplotlib()", 332 'pylab': "__check_pylab()", 333 'pylab plottable': "__check_pylab_plottable()", 334 'openopt': "import scikits.openopt as __", 335 'mdp': "import mdp as __", 336 'mdp ge 2.4': "from mdp.nodes import LLENode as __", 337 'sg_fixedcachesize': "__check_shogun(3043, [2456])", 338 # 3318 corresponds to release 0.6.4 339 'sg ge 0.6.4': "__check_shogun(3318)", 340 'hcluster': "import hcluster as __", 341 'griddata': "__check_griddata()", 342 'cPickle': "import cPickle as __", 343 'gzip': "import gzip as __", 344 'lxml': "from lxml import objectify as __", 345 'atlas_pymvpa': "__check_atlas_family('pymvpa')", 346 'atlas_fsl': "__check_atlas_family('fsl')", 347 'running ipython env': "__check_in_ipython()", 348 'reportlab': "__check_reportlab()", 349 } 350 351
352 -def exists(dep, force=False, raiseException=False, issueWarning=None):
353 """ 354 Test whether a known dependency is installed on the system. 355 356 This method allows us to test for individual dependencies without 357 testing all known dependencies. It also ensures that we only test 358 for a dependency once. 359 360 :Parameters: 361 dep : string or list of string 362 The dependency key(s) to test. 363 force : boolean 364 Whether to force the test even if it has already been 365 performed. 366 raiseException : boolean 367 Whether to raise RuntimeError if dependency is missing. 368 issueWarning : string or None or True 369 If string, warning with given message would be thrown. 370 If True, standard message would be used for the warning 371 text. 372 """ 373 # if we are provided with a list of deps - go through all of them 374 if isinstance(dep, list) or isinstance(dep, tuple): 375 results = [ exists(dep_, force, raiseException) for dep_ in dep ] 376 return bool(reduce(lambda x,y: x and y, results, True)) 377 378 # where to look in cfg 379 cfgid = 'have ' + dep 380 381 # prevent unnecessarry testing 382 if cfg.has_option('externals', cfgid) \ 383 and not cfg.getboolean('externals', 'retest', default='no') \ 384 and not force: 385 if __debug__: 386 debug('EXT', "Skip retesting for '%s'." % dep) 387 388 # check whether an exception should be raised, even though the external 389 # was already tested previously 390 if not cfg.getboolean('externals', cfgid) \ 391 and raiseException \ 392 and cfg.getboolean('externals', 'raise exception', True): 393 raise RuntimeError, "Required external '%s' was not found" % dep 394 return cfg.getboolean('externals', cfgid) 395 396 397 # determine availability of external (non-cached) 398 399 # default to 'not found' 400 result = False 401 402 if not _KNOWN.has_key(dep): 403 raise ValueError, "%s is not a known dependency key." % (dep) 404 else: 405 # try and load the specific dependency 406 if __debug__: 407 debug('EXT', "Checking for the presence of %s" % dep) 408 409 # Exceptions which are silently caught while running tests for externals 410 _caught_exceptions = [ImportError, AttributeError, RuntimeError] 411 412 # check whether RPy is involved and catch its excpetions as well. 413 # however, try to determine whether this is really necessary, as 414 # importing RPy also involved starting a full-blown R session, which can 415 # take seconds and therefore is quite nasty... 416 if dep.count('rpy') or _KNOWN[dep].count('rpy'): 417 try: 418 if dep == 'rpy': 419 __check_rpy() # needed to be run to adjust options first 420 else: 421 if exists('rpy'): 422 # otherwise no need to add anything -- test 423 # would fail since rpy isn't available 424 from rpy import RException 425 _caught_exceptions += [RException] 426 except: 427 pass 428 429 430 estr = '' 431 try: 432 exec _KNOWN[dep] 433 result = True 434 except tuple(_caught_exceptions), e: 435 estr = ". Caught exception was: " + str(e) 436 437 if __debug__: 438 debug('EXT', "Presence of %s is%s verified%s" % 439 (dep, {True:'', False:' NOT'}[result], estr)) 440 441 if not result: 442 if raiseException \ 443 and cfg.getboolean('externals', 'raise exception', True): 444 raise RuntimeError, "Required external '%s' was not found" % dep 445 if issueWarning is not None \ 446 and cfg.getboolean('externals', 'issue warning', True): 447 if issueWarning is True: 448 warning("Required external '%s' was not found" % dep) 449 else: 450 warning(issueWarning) 451 452 453 # store result in config manager 454 if not cfg.has_section('externals'): 455 cfg.add_section('externals') 456 if result: 457 cfg.set('externals', 'have ' + dep, 'yes') 458 else: 459 cfg.set('externals', 'have ' + dep, 'no') 460 461 return result
462 463
464 -def testAllDependencies(force=False):
465 """ 466 Test for all known dependencies. 467 468 :Parameters: 469 force : boolean 470 Whether to force the test even if it has already been 471 performed. 472 473 """ 474 # loop over all known dependencies 475 for dep in _KNOWN: 476 if not exists(dep, force): 477 warning("%s is not available." % dep) 478 479 if __debug__: 480 debug('EXT', 'The following optional externals are present: %s' \ 481 % [ k[5:] for k in cfg.options('externals') 482 if k.startswith('have') \ 483 and cfg.getboolean('externals', k) == True ])
484