util.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. """Miscellaneous utility functions and classes.
  2. This module is used internally by Tornado. It is not necessarily expected
  3. that the functions and classes defined here will be useful to other
  4. applications, but they are documented here in case they are.
  5. The one public-facing part of this module is the `Configurable` class
  6. and its `~Configurable.configure` method, which becomes a part of the
  7. interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`,
  8. and `.Resolver`.
  9. """
  10. import array
  11. import atexit
  12. from inspect import getfullargspec
  13. import os
  14. import re
  15. import typing
  16. import zlib
  17. from typing import (
  18. Any,
  19. Optional,
  20. Dict,
  21. Mapping,
  22. List,
  23. Tuple,
  24. Match,
  25. Callable,
  26. Type,
  27. Sequence,
  28. )
  29. if typing.TYPE_CHECKING:
  30. # Additional imports only used in type comments.
  31. # This lets us make these imports lazy.
  32. import datetime # noqa: F401
  33. from types import TracebackType # noqa: F401
  34. from typing import Union # noqa: F401
  35. import unittest # noqa: F401
  36. # Aliases for types that are spelled differently in different Python
  37. # versions. bytes_type is deprecated and no longer used in Tornado
  38. # itself but is left in case anyone outside Tornado is using it.
  39. bytes_type = bytes
  40. unicode_type = str
  41. basestring_type = str
  42. try:
  43. from sys import is_finalizing
  44. except ImportError:
  45. # Emulate it
  46. def _get_emulated_is_finalizing() -> Callable[[], bool]:
  47. L = [] # type: List[None]
  48. atexit.register(lambda: L.append(None))
  49. def is_finalizing() -> bool:
  50. # Not referencing any globals here
  51. return L != []
  52. return is_finalizing
  53. is_finalizing = _get_emulated_is_finalizing()
  54. class TimeoutError(Exception):
  55. """Exception raised by `.with_timeout` and `.IOLoop.run_sync`.
  56. .. versionchanged:: 5.0:
  57. Unified ``tornado.gen.TimeoutError`` and
  58. ``tornado.ioloop.TimeoutError`` as ``tornado.util.TimeoutError``.
  59. Both former names remain as aliases.
  60. """
  61. class ObjectDict(Dict[str, Any]):
  62. """Makes a dictionary behave like an object, with attribute-style access.
  63. """
  64. def __getattr__(self, name: str) -> Any:
  65. try:
  66. return self[name]
  67. except KeyError:
  68. raise AttributeError(name)
  69. def __setattr__(self, name: str, value: Any) -> None:
  70. self[name] = value
  71. class GzipDecompressor(object):
  72. """Streaming gzip decompressor.
  73. The interface is like that of `zlib.decompressobj` (without some of the
  74. optional arguments, but it understands gzip headers and checksums.
  75. """
  76. def __init__(self) -> None:
  77. # Magic parameter makes zlib module understand gzip header
  78. # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
  79. # This works on cpython and pypy, but not jython.
  80. self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
  81. def decompress(self, value: bytes, max_length: int = 0) -> bytes:
  82. """Decompress a chunk, returning newly-available data.
  83. Some data may be buffered for later processing; `flush` must
  84. be called when there is no more input data to ensure that
  85. all data was processed.
  86. If ``max_length`` is given, some input data may be left over
  87. in ``unconsumed_tail``; you must retrieve this value and pass
  88. it back to a future call to `decompress` if it is not empty.
  89. """
  90. return self.decompressobj.decompress(value, max_length)
  91. @property
  92. def unconsumed_tail(self) -> bytes:
  93. """Returns the unconsumed portion left over
  94. """
  95. return self.decompressobj.unconsumed_tail
  96. def flush(self) -> bytes:
  97. """Return any remaining buffered data not yet returned by decompress.
  98. Also checks for errors such as truncated input.
  99. No other methods may be called on this object after `flush`.
  100. """
  101. return self.decompressobj.flush()
  102. def import_object(name: str) -> Any:
  103. """Imports an object by name.
  104. ``import_object('x')`` is equivalent to ``import x``.
  105. ``import_object('x.y.z')`` is equivalent to ``from x.y import z``.
  106. >>> import tornado.escape
  107. >>> import_object('tornado.escape') is tornado.escape
  108. True
  109. >>> import_object('tornado.escape.utf8') is tornado.escape.utf8
  110. True
  111. >>> import_object('tornado') is tornado
  112. True
  113. >>> import_object('tornado.missing_module')
  114. Traceback (most recent call last):
  115. ...
  116. ImportError: No module named missing_module
  117. """
  118. if name.count(".") == 0:
  119. return __import__(name)
  120. parts = name.split(".")
  121. obj = __import__(".".join(parts[:-1]), fromlist=[parts[-1]])
  122. try:
  123. return getattr(obj, parts[-1])
  124. except AttributeError:
  125. raise ImportError("No module named %s" % parts[-1])
  126. def exec_in(code: Any, glob: Dict[str, Any], loc: Mapping[str, Any] = None) -> None:
  127. if isinstance(code, str):
  128. # exec(string) inherits the caller's future imports; compile
  129. # the string first to prevent that.
  130. code = compile(code, "<string>", "exec", dont_inherit=True)
  131. exec(code, glob, loc)
  132. def raise_exc_info(
  133. exc_info, # type: Tuple[Optional[type], Optional[BaseException], Optional[TracebackType]]
  134. ):
  135. # type: (...) -> typing.NoReturn
  136. #
  137. # This function's type annotation must use comments instead of
  138. # real annotations because typing.NoReturn does not exist in
  139. # python 3.5's typing module. The formatting is funky because this
  140. # is apparently what flake8 wants.
  141. try:
  142. if exc_info[1] is not None:
  143. raise exc_info[1].with_traceback(exc_info[2])
  144. else:
  145. raise TypeError("raise_exc_info called with no exception")
  146. finally:
  147. # Clear the traceback reference from our stack frame to
  148. # minimize circular references that slow down GC.
  149. exc_info = (None, None, None)
  150. def errno_from_exception(e: BaseException) -> Optional[int]:
  151. """Provides the errno from an Exception object.
  152. There are cases that the errno attribute was not set so we pull
  153. the errno out of the args but if someone instantiates an Exception
  154. without any args you will get a tuple error. So this function
  155. abstracts all that behavior to give you a safe way to get the
  156. errno.
  157. """
  158. if hasattr(e, "errno"):
  159. return e.errno # type: ignore
  160. elif e.args:
  161. return e.args[0]
  162. else:
  163. return None
  164. _alphanum = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
  165. def _re_unescape_replacement(match: Match[str]) -> str:
  166. group = match.group(1)
  167. if group[0] in _alphanum:
  168. raise ValueError("cannot unescape '\\\\%s'" % group[0])
  169. return group
  170. _re_unescape_pattern = re.compile(r"\\(.)", re.DOTALL)
  171. def re_unescape(s: str) -> str:
  172. r"""Unescape a string escaped by `re.escape`.
  173. May raise ``ValueError`` for regular expressions which could not
  174. have been produced by `re.escape` (for example, strings containing
  175. ``\d`` cannot be unescaped).
  176. .. versionadded:: 4.4
  177. """
  178. return _re_unescape_pattern.sub(_re_unescape_replacement, s)
  179. class Configurable(object):
  180. """Base class for configurable interfaces.
  181. A configurable interface is an (abstract) class whose constructor
  182. acts as a factory function for one of its implementation subclasses.
  183. The implementation subclass as well as optional keyword arguments to
  184. its initializer can be set globally at runtime with `configure`.
  185. By using the constructor as the factory method, the interface
  186. looks like a normal class, `isinstance` works as usual, etc. This
  187. pattern is most useful when the choice of implementation is likely
  188. to be a global decision (e.g. when `~select.epoll` is available,
  189. always use it instead of `~select.select`), or when a
  190. previously-monolithic class has been split into specialized
  191. subclasses.
  192. Configurable subclasses must define the class methods
  193. `configurable_base` and `configurable_default`, and use the instance
  194. method `initialize` instead of ``__init__``.
  195. .. versionchanged:: 5.0
  196. It is now possible for configuration to be specified at
  197. multiple levels of a class hierarchy.
  198. """
  199. # Type annotations on this class are mostly done with comments
  200. # because they need to refer to Configurable, which isn't defined
  201. # until after the class definition block. These can use regular
  202. # annotations when our minimum python version is 3.7.
  203. #
  204. # There may be a clever way to use generics here to get more
  205. # precise types (i.e. for a particular Configurable subclass T,
  206. # all the types are subclasses of T, not just Configurable).
  207. __impl_class = None # type: Optional[Type[Configurable]]
  208. __impl_kwargs = None # type: Dict[str, Any]
  209. def __new__(cls, *args: Any, **kwargs: Any) -> Any:
  210. base = cls.configurable_base()
  211. init_kwargs = {} # type: Dict[str, Any]
  212. if cls is base:
  213. impl = cls.configured_class()
  214. if base.__impl_kwargs:
  215. init_kwargs.update(base.__impl_kwargs)
  216. else:
  217. impl = cls
  218. init_kwargs.update(kwargs)
  219. if impl.configurable_base() is not base:
  220. # The impl class is itself configurable, so recurse.
  221. return impl(*args, **init_kwargs)
  222. instance = super(Configurable, cls).__new__(impl)
  223. # initialize vs __init__ chosen for compatibility with AsyncHTTPClient
  224. # singleton magic. If we get rid of that we can switch to __init__
  225. # here too.
  226. instance.initialize(*args, **init_kwargs)
  227. return instance
  228. @classmethod
  229. def configurable_base(cls):
  230. # type: () -> Type[Configurable]
  231. """Returns the base class of a configurable hierarchy.
  232. This will normally return the class in which it is defined.
  233. (which is *not* necessarily the same as the ``cls`` classmethod
  234. parameter).
  235. """
  236. raise NotImplementedError()
  237. @classmethod
  238. def configurable_default(cls):
  239. # type: () -> Type[Configurable]
  240. """Returns the implementation class to be used if none is configured."""
  241. raise NotImplementedError()
  242. def _initialize(self) -> None:
  243. pass
  244. initialize = _initialize # type: Callable[..., None]
  245. """Initialize a `Configurable` subclass instance.
  246. Configurable classes should use `initialize` instead of ``__init__``.
  247. .. versionchanged:: 4.2
  248. Now accepts positional arguments in addition to keyword arguments.
  249. """
  250. @classmethod
  251. def configure(cls, impl, **kwargs):
  252. # type: (Union[None, str, Type[Configurable]], Any) -> None
  253. """Sets the class to use when the base class is instantiated.
  254. Keyword arguments will be saved and added to the arguments passed
  255. to the constructor. This can be used to set global defaults for
  256. some parameters.
  257. """
  258. base = cls.configurable_base()
  259. if isinstance(impl, str):
  260. impl = typing.cast(Type[Configurable], import_object(impl))
  261. if impl is not None and not issubclass(impl, cls):
  262. raise ValueError("Invalid subclass of %s" % cls)
  263. base.__impl_class = impl
  264. base.__impl_kwargs = kwargs
  265. @classmethod
  266. def configured_class(cls):
  267. # type: () -> Type[Configurable]
  268. """Returns the currently configured class."""
  269. base = cls.configurable_base()
  270. # Manually mangle the private name to see whether this base
  271. # has been configured (and not another base higher in the
  272. # hierarchy).
  273. if base.__dict__.get("_Configurable__impl_class") is None:
  274. base.__impl_class = cls.configurable_default()
  275. if base.__impl_class is not None:
  276. return base.__impl_class
  277. else:
  278. # Should be impossible, but mypy wants an explicit check.
  279. raise ValueError("configured class not found")
  280. @classmethod
  281. def _save_configuration(cls):
  282. # type: () -> Tuple[Optional[Type[Configurable]], Dict[str, Any]]
  283. base = cls.configurable_base()
  284. return (base.__impl_class, base.__impl_kwargs)
  285. @classmethod
  286. def _restore_configuration(cls, saved):
  287. # type: (Tuple[Optional[Type[Configurable]], Dict[str, Any]]) -> None
  288. base = cls.configurable_base()
  289. base.__impl_class = saved[0]
  290. base.__impl_kwargs = saved[1]
  291. class ArgReplacer(object):
  292. """Replaces one value in an ``args, kwargs`` pair.
  293. Inspects the function signature to find an argument by name
  294. whether it is passed by position or keyword. For use in decorators
  295. and similar wrappers.
  296. """
  297. def __init__(self, func: Callable, name: str) -> None:
  298. self.name = name
  299. try:
  300. self.arg_pos = self._getargnames(func).index(name) # type: Optional[int]
  301. except ValueError:
  302. # Not a positional parameter
  303. self.arg_pos = None
  304. def _getargnames(self, func: Callable) -> List[str]:
  305. try:
  306. return getfullargspec(func).args
  307. except TypeError:
  308. if hasattr(func, "func_code"):
  309. # Cython-generated code has all the attributes needed
  310. # by inspect.getfullargspec, but the inspect module only
  311. # works with ordinary functions. Inline the portion of
  312. # getfullargspec that we need here. Note that for static
  313. # functions the @cython.binding(True) decorator must
  314. # be used (for methods it works out of the box).
  315. code = func.func_code # type: ignore
  316. return code.co_varnames[: code.co_argcount]
  317. raise
  318. def get_old_value(
  319. self, args: Sequence[Any], kwargs: Dict[str, Any], default: Any = None
  320. ) -> Any:
  321. """Returns the old value of the named argument without replacing it.
  322. Returns ``default`` if the argument is not present.
  323. """
  324. if self.arg_pos is not None and len(args) > self.arg_pos:
  325. return args[self.arg_pos]
  326. else:
  327. return kwargs.get(self.name, default)
  328. def replace(
  329. self, new_value: Any, args: Sequence[Any], kwargs: Dict[str, Any]
  330. ) -> Tuple[Any, Sequence[Any], Dict[str, Any]]:
  331. """Replace the named argument in ``args, kwargs`` with ``new_value``.
  332. Returns ``(old_value, args, kwargs)``. The returned ``args`` and
  333. ``kwargs`` objects may not be the same as the input objects, or
  334. the input objects may be mutated.
  335. If the named argument was not found, ``new_value`` will be added
  336. to ``kwargs`` and None will be returned as ``old_value``.
  337. """
  338. if self.arg_pos is not None and len(args) > self.arg_pos:
  339. # The arg to replace is passed positionally
  340. old_value = args[self.arg_pos]
  341. args = list(args) # *args is normally a tuple
  342. args[self.arg_pos] = new_value
  343. else:
  344. # The arg to replace is either omitted or passed by keyword.
  345. old_value = kwargs.get(self.name)
  346. kwargs[self.name] = new_value
  347. return old_value, args, kwargs
  348. def timedelta_to_seconds(td):
  349. # type: (datetime.timedelta) -> float
  350. """Equivalent to ``td.total_seconds()`` (introduced in Python 2.7)."""
  351. return td.total_seconds()
  352. def _websocket_mask_python(mask: bytes, data: bytes) -> bytes:
  353. """Websocket masking function.
  354. `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length.
  355. Returns a `bytes` object of the same length as `data` with the mask applied
  356. as specified in section 5.3 of RFC 6455.
  357. This pure-python implementation may be replaced by an optimized version when available.
  358. """
  359. mask_arr = array.array("B", mask)
  360. unmasked_arr = array.array("B", data)
  361. for i in range(len(data)):
  362. unmasked_arr[i] = unmasked_arr[i] ^ mask_arr[i % 4]
  363. return unmasked_arr.tobytes()
  364. if os.environ.get("TORNADO_NO_EXTENSION") or os.environ.get("TORNADO_EXTENSION") == "0":
  365. # These environment variables exist to make it easier to do performance
  366. # comparisons; they are not guaranteed to remain supported in the future.
  367. _websocket_mask = _websocket_mask_python
  368. else:
  369. try:
  370. from tornado.speedups import websocket_mask as _websocket_mask
  371. except ImportError:
  372. if os.environ.get("TORNADO_EXTENSION") == "1":
  373. raise
  374. _websocket_mask = _websocket_mask_python
  375. def doctests():
  376. # type: () -> unittest.TestSuite
  377. import doctest
  378. return doctest.DocTestSuite()