jupytersketcher/Lib/site-packages/joblib/externals/loky/_base.py

###############################################################################
# Backport concurrent.futures for python2.7/3.3
#
# author: Thomas Moreau and Olivier Grisel
#
# adapted from concurrent/futures/_base.py (17/02/2017)
#  * Do not use yield from
#  * Use old super syntax
#
# Copyright 2009 Brian Quinlan. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

import sys
import time
import logging
import threading
import collections


if sys.version_info[:2] >= (3, 3):

    from concurrent.futures import wait, as_completed
    from concurrent.futures import TimeoutError, CancelledError
    from concurrent.futures import Executor, Future as _BaseFuture

    from concurrent.futures import FIRST_EXCEPTION
    from concurrent.futures import ALL_COMPLETED, FIRST_COMPLETED

    from concurrent.futures._base import LOGGER
    from concurrent.futures._base import PENDING, RUNNING, CANCELLED
    from concurrent.futures._base import CANCELLED_AND_NOTIFIED, FINISHED
else:

    FIRST_COMPLETED = 'FIRST_COMPLETED'
    FIRST_EXCEPTION = 'FIRST_EXCEPTION'
    ALL_COMPLETED = 'ALL_COMPLETED'
    _AS_COMPLETED = '_AS_COMPLETED'

    # Possible future states (for internal use by the futures package).
    PENDING = 'PENDING'
    RUNNING = 'RUNNING'
    # The future was cancelled by the user...
    CANCELLED = 'CANCELLED'
    # ...and _Waiter.add_cancelled() was called by a worker.
    CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
    FINISHED = 'FINISHED'

    _FUTURE_STATES = [
        PENDING,
        RUNNING,
        CANCELLED,
        CANCELLED_AND_NOTIFIED,
        FINISHED
    ]

    _STATE_TO_DESCRIPTION_MAP = {
        PENDING: "pending",
        RUNNING: "running",
        CANCELLED: "cancelled",
        CANCELLED_AND_NOTIFIED: "cancelled",
        FINISHED: "finished"
    }

    # Logger for internal use by the futures package.
    LOGGER = logging.getLogger("concurrent.futures")

    class Error(Exception):
        """Base class for all future-related exceptions."""
        pass

    class CancelledError(Error):
        """The Future was cancelled."""
        pass

    class TimeoutError(Error):
        """The operation exceeded the given deadline."""
        pass

    class _Waiter(object):
        """Provides the event that wait() and as_completed() block on."""
        def __init__(self):
            self.event = threading.Event()
            self.finished_futures = []

        def add_result(self, future):
            self.finished_futures.append(future)

        def add_exception(self, future):
            self.finished_futures.append(future)

        def add_cancelled(self, future):
            self.finished_futures.append(future)

    class _AsCompletedWaiter(_Waiter):
        """Used by as_completed()."""

        def __init__(self):
            super(_AsCompletedWaiter, self).__init__()
            self.lock = threading.Lock()

        def add_result(self, future):
            with self.lock:
                super(_AsCompletedWaiter, self).add_result(future)
                self.event.set()

        def add_exception(self, future):
            with self.lock:
                super(_AsCompletedWaiter, self).add_exception(future)
                self.event.set()

        def add_cancelled(self, future):
            with self.lock:
                super(_AsCompletedWaiter, self).add_cancelled(future)
                self.event.set()

    class _FirstCompletedWaiter(_Waiter):
        """Used by wait(return_when=FIRST_COMPLETED)."""

        def add_result(self, future):
            super(_FirstCompletedWaiter, self).add_result(future)
            self.event.set()

        def add_exception(self, future):
            super(_FirstCompletedWaiter, self).add_exception(future)
            self.event.set()

        def add_cancelled(self, future):
            super(_FirstCompletedWaiter, self).add_cancelled(future)
            self.event.set()

    class _AllCompletedWaiter(_Waiter):
        """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""

        def __init__(self, num_pending_calls, stop_on_exception):
            self.num_pending_calls = num_pending_calls
            self.stop_on_exception = stop_on_exception
            self.lock = threading.Lock()
            super(_AllCompletedWaiter, self).__init__()

        def _decrement_pending_calls(self):
            with self.lock:
                self.num_pending_calls -= 1
                if not self.num_pending_calls:
                    self.event.set()

        def add_result(self, future):
            super(_AllCompletedWaiter, self).add_result(future)
            self._decrement_pending_calls()

        def add_exception(self, future):
            super(_AllCompletedWaiter, self).add_exception(future)
            if self.stop_on_exception:
                self.event.set()
            else:
                self._decrement_pending_calls()

        def add_cancelled(self, future):
            super(_AllCompletedWaiter, self).add_cancelled(future)
            self._decrement_pending_calls()

    class _AcquireFutures(object):
        """A context manager that does an ordered acquire of Future conditions.
        """

        def __init__(self, futures):
            self.futures = sorted(futures, key=id)

        def __enter__(self):
            for future in self.futures:
                future._condition.acquire()

        def __exit__(self, *args):
            for future in self.futures:
                future._condition.release()

    def _create_and_install_waiters(fs, return_when):
        if return_when == _AS_COMPLETED:
            waiter = _AsCompletedWaiter()
        elif return_when == FIRST_COMPLETED:
            waiter = _FirstCompletedWaiter()
        else:
            pending_count = sum(
                    f._state not in [CANCELLED_AND_NOTIFIED, FINISHED]
                    for f in fs)

            if return_when == FIRST_EXCEPTION:
                waiter = _AllCompletedWaiter(pending_count,
                                             stop_on_exception=True)
            elif return_when == ALL_COMPLETED:
                waiter = _AllCompletedWaiter(pending_count,
                                             stop_on_exception=False)
            else:
                raise ValueError("Invalid return condition: %r" % return_when)

        for f in fs:
            f._waiters.append(waiter)

        return waiter

    def as_completed(fs, timeout=None):
        """An iterator over the given futures that yields each as it completes.

        Args:
            fs: The sequence of Futures (possibly created by different
                Executors) to iterate over.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.

        Returns:
            An iterator that yields the given Futures as they complete
            (finished or cancelled). If any given Futures are duplicated, they
            will be returned once.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
        """
        if timeout is not None:
            end_time = timeout + time.time()

        fs = set(fs)
        with _AcquireFutures(fs):
            finished = set(
                    f for f in fs
                    if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
            pending = fs - finished
            waiter = _create_and_install_waiters(fs, _AS_COMPLETED)

        try:
            for future in finished:
                yield future

            while pending:
                if timeout is None:
                    wait_timeout = None
                else:
                    wait_timeout = end_time - time.time()
                    if wait_timeout < 0:
                        raise TimeoutError('%d (of %d) futures unfinished' % (
                            len(pending), len(fs)))

                waiter.event.wait(wait_timeout)

                with waiter.lock:
                    finished = waiter.finished_futures
                    waiter.finished_futures = []
                    waiter.event.clear()

                for future in finished:
                    yield future
                    pending.remove(future)

        finally:
            for f in fs:
                with f._condition:
                    f._waiters.remove(waiter)

    DoneAndNotDoneFutures = collections.namedtuple(
            'DoneAndNotDoneFutures', 'done not_done')

    def wait(fs, timeout=None, return_when=ALL_COMPLETED):
        """Wait for the futures in the given sequence to complete.

        Args:
            fs: The sequence of Futures (possibly created by different
                Executors) to wait upon.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.
            return_when: Indicates when this function should return. The
                options are:

                FIRST_COMPLETED - Return when any future finishes or is
                                cancelled.
                FIRST_EXCEPTION - Return when any future finishes by raising an
                                exception. If no future raises an exception
                                then it is equivalent to ALL_COMPLETED.
                ALL_COMPLETED -   Return when all futures finish or are
                                cancelled.

        Returns:
            A named 2-tuple of sets. The first set, named 'done', contains the
            futures that completed (is finished or cancelled) before the wait
            completed. The second set, named 'not_done', contains uncompleted
            futures.
        """
        with _AcquireFutures(fs):
            done = set(f for f in fs
                       if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
            not_done = set(fs) - done

            if (return_when == FIRST_COMPLETED) and done:
                return DoneAndNotDoneFutures(done, not_done)
            elif (return_when == FIRST_EXCEPTION) and done:
                if any(f for f in done
                       if not f.cancelled() and f.exception() is not None):
                    return DoneAndNotDoneFutures(done, not_done)

            if len(done) == len(fs):
                return DoneAndNotDoneFutures(done, not_done)

            waiter = _create_and_install_waiters(fs, return_when)

        waiter.event.wait(timeout)
        for f in fs:
            with f._condition:
                f._waiters.remove(waiter)

        done.update(waiter.finished_futures)
        return DoneAndNotDoneFutures(done, set(fs) - done)

    class _BaseFuture(object):
        """Represents the result of an asynchronous computation."""

        def __init__(self):
            """Initializes the future. Should not be called by clients."""
            self._condition = threading.Condition()
            self._state = PENDING
            self._result = None
            self._exception = None
            self._waiters = []
            self._done_callbacks = []

        def __repr__(self):
            with self._condition:
                if self._state == FINISHED:
                    if self._exception:
                        return '<%s at %#x state=%s raised %s>' % (
                            self.__class__.__name__,
                            id(self),
                            _STATE_TO_DESCRIPTION_MAP[self._state],
                            self._exception.__class__.__name__)
                    else:
                        return '<%s at %#x state=%s returned %s>' % (
                            self.__class__.__name__,
                            id(self),
                            _STATE_TO_DESCRIPTION_MAP[self._state],
                            self._result.__class__.__name__)
                return '<%s at %#x state=%s>' % (
                        self.__class__.__name__,
                        id(self),
                        _STATE_TO_DESCRIPTION_MAP[self._state])

        def cancel(self):
            """Cancel the future if possible.

            Returns True if the future was cancelled, False otherwise. A future
            cannot be cancelled if it is running or has already completed.
            """
            with self._condition:
                if self._state in [RUNNING, FINISHED]:
                    return False

                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
                    return True

                self._state = CANCELLED
                self._condition.notify_all()

            self._invoke_callbacks()
            return True

        def cancelled(self):
            """Return True if the future was cancelled."""
            with self._condition:
                return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]

        def running(self):
            """Return True if the future is currently executing."""
            with self._condition:
                return self._state == RUNNING

        def done(self):
            """Return True of the future was cancelled or finished executing.
            """
            with self._condition:
                return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED,
                                       FINISHED]

        def __get_result(self):
            if self._exception:
                raise self._exception
            else:
                return self._result

        def add_done_callback(self, fn):
            """Attaches a callable that will be called when the future finishes.

            Args:
                fn: A callable that will be called with this future as its only
                    argument when the future completes or is cancelled. The
                    callable will always be called by a thread in the same
                    process in which it was added. If the future has already
                    completed or been cancelled then the callable will be
                    called immediately. These callables are called in the order
                    that they were added.
            """
            with self._condition:
                if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED,
                                       FINISHED]:
                    self._done_callbacks.append(fn)
                    return
            fn(self)

        def result(self, timeout=None):
            """Return the result of the call that the future represents.

            Args:
                timeout: The number of seconds to wait for the result if the
                    future isn't done. If None, then there is no limit on the
                    wait time.

            Returns:
                The result of the call that the future represents.

            Raises:
                CancelledError: If the future was cancelled.
                TimeoutError: If the future didn't finish executing before the
                    given timeout.
                Exception: If the call raised then that exception will be
                raised.
            """
            with self._condition:
                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
                    raise CancelledError()
                elif self._state == FINISHED:
                    return self.__get_result()

                self._condition.wait(timeout)

                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
                    raise CancelledError()
                elif self._state == FINISHED:
                    return self.__get_result()
                else:
                    raise TimeoutError()

        def exception(self, timeout=None):
            """Return the exception raised by the call that the future
            represents.

            Args:
                timeout: The number of seconds to wait for the exception if the
                    future isn't done. If None, then there is no limit on the
                    wait time.

            Returns:
                The exception raised by the call that the future represents or
                None if the call completed without raising.

            Raises:
                CancelledError: If the future was cancelled.
                TimeoutError: If the future didn't finish executing before the
                    given timeout.
            """

            with self._condition:
                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
                    raise CancelledError()
                elif self._state == FINISHED:
                    return self._exception

                self._condition.wait(timeout)

                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
                    raise CancelledError()
                elif self._state == FINISHED:
                    return self._exception
                else:
                    raise TimeoutError()

        # The following methods should only be used by Executors and in tests.
        def set_running_or_notify_cancel(self):
            """Mark the future as running or process any cancel notifications.

            Should only be used by Executor implementations and unit tests.

            If the future has been cancelled (cancel() was called and returned
            True) then any threads waiting on the future completing (though
            calls to as_completed() or wait()) are notified and False is
            returned.

            If the future was not cancelled then it is put in the running state
            (future calls to running() will return True) and True is returned.

            This method should be called by Executor implementations before
            executing the work associated with this future. If this method
            returns False then the work should not be executed.

            Returns:
                False if the Future was cancelled, True otherwise.

            Raises:
                RuntimeError: if this method was already called or if
                    set_result() or set_exception() was called.
            """
            with self._condition:
                if self._state == CANCELLED:
                    self._state = CANCELLED_AND_NOTIFIED
                    for waiter in self._waiters:
                        waiter.add_cancelled(self)
                    # self._condition.notify_all() is not necessary because
                    # self.cancel() triggers a notification.
                    return False
                elif self._state == PENDING:
                    self._state = RUNNING
                    return True
                else:
                    LOGGER.critical('Future %s in unexpected state: %s',
                                    id(self),
                                    self._state)
                    raise RuntimeError('Future in unexpected state')

        def set_result(self, result):
            """Sets the return value of work associated with the future.

            Should only be used by Executor implementations and unit tests.
            """
            with self._condition:
                self._result = result
                self._state = FINISHED
                for waiter in self._waiters:
                    waiter.add_result(self)
                self._condition.notify_all()
            self._invoke_callbacks()

        def set_exception(self, exception):
            """Sets the result of the future as being the given exception.

            Should only be used by Executor implementations and unit tests.
            """
            with self._condition:
                self._exception = exception
                self._state = FINISHED
                for waiter in self._waiters:
                    waiter.add_exception(self)
                self._condition.notify_all()
            self._invoke_callbacks()

    class Executor(object):
        """This is an abstract base class for concrete asynchronous executors.
        """

        def submit(self, fn, *args, **kwargs):
            """Submits a callable to be executed with the given arguments.

            Schedules the callable to be executed as fn(*args, **kwargs) and
            returns a Future instance representing the execution of the
            callable.

            Returns:
                A Future representing the given call.
            """
            raise NotImplementedError()

        def map(self, fn, *iterables, **kwargs):
            """Returns an iterator equivalent to map(fn, iter).

            Args:
                fn: A callable that will take as many arguments as there are
                    passed iterables.
                timeout: The maximum number of seconds to wait. If None, then
                    there is no limit on the wait time.
                chunksize: The size of the chunks the iterable will be broken
                    into before being passed to a child process. This argument
                    is only used by ProcessPoolExecutor; it is ignored by
                    ThreadPoolExecutor.

            Returns:
                An iterator equivalent to: map(func, *iterables) but the calls
                may be evaluated out-of-order.

            Raises:
                TimeoutError: If the entire result iterator could not be
                    generated before the given timeout.
                Exception: If fn(*args) raises for any values.
            """
            timeout = kwargs.get('timeout')
            if timeout is not None:
                end_time = timeout + time.time()

            fs = [self.submit(fn, *args) for args in zip(*iterables)]

            # Yield must be hidden in closure so that the futures are submitted
            # before the first iterator value is required.
            def result_iterator():
                try:
                    for future in fs:
                        if timeout is None:
                            yield future.result()
                        else:
                            yield future.result(end_time - time.time())
                finally:
                    for future in fs:
                        future.cancel()
            return result_iterator()

        def shutdown(self, wait=True):
            """Clean-up the resources associated with the Executor.

            It is safe to call this method several times. Otherwise, no other
            methods can be called after this one.

            Args:
                wait: If True then shutdown will not return until all running
                    futures have finished executing and the resources used by
                    the executor have been reclaimed.
            """
            pass

        def __enter__(self):
            return self

        def __exit__(self, exc_type, exc_val, exc_tb):
            self.shutdown(wait=True)
            return False


# To make loky._base.Future instances awaitable  by concurrent.futures.wait,
# derive our custom Future class from _BaseFuture. _invoke_callback is the only
# modification made to this class in loky.
class Future(_BaseFuture):
    def _invoke_callbacks(self):
        for callback in self._done_callbacks:
            try:
                callback(self)
            except BaseException:
                LOGGER.exception('exception calling callback for %r', self)