_multiprocessing_helpers.py 1.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152
  1. """Helper module to factorize the conditional multiprocessing import logic
  2. We use a distinct module to simplify import statements and avoid introducing
  3. circular dependencies (for instance for the assert_spawning name).
  4. """
  5. import os
  6. import warnings
  7. # Obtain possible configuration from the environment, assuming 1 (on)
  8. # by default, upon 0 set to None. Should instructively fail if some non
  9. # 0/1 value is set.
  10. mp = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None
  11. if mp:
  12. try:
  13. import multiprocessing as mp
  14. except ImportError:
  15. mp = None
  16. # 2nd stage: validate that locking is available on the system and
  17. # issue a warning if not
  18. if mp is not None:
  19. try:
  20. # try to create a named semaphore using SemLock to make sure they are
  21. # available on this platform. We use the low level object
  22. # _multiprocessing.SemLock to avoid spawning a resource tracker on
  23. # Unix system or changing the default backend.
  24. import tempfile
  25. from _multiprocessing import SemLock
  26. _rand = tempfile._RandomNameSequence()
  27. for i in range(100):
  28. try:
  29. name = '/joblib-{}-{}' .format(
  30. os.getpid(), next(_rand))
  31. _sem = SemLock(0, 0, 1, name=name, unlink=True)
  32. del _sem # cleanup
  33. break
  34. except FileExistsError as e: # pragma: no cover
  35. if i >= 99:
  36. raise FileExistsError(
  37. 'cannot find name for semaphore') from e
  38. except (FileExistsError, AttributeError, ImportError, OSError) as e:
  39. mp = None
  40. warnings.warn('%s. joblib will operate in serial mode' % (e,))
  41. # 3rd stage: backward compat for the assert_spawning helper
  42. if mp is not None:
  43. from multiprocessing.context import assert_spawning
  44. else:
  45. assert_spawning = None