simple_httpclient_test.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. import collections
  2. from contextlib import closing
  3. import errno
  4. import gzip
  5. import logging
  6. import os
  7. import re
  8. import socket
  9. import ssl
  10. import sys
  11. import typing # noqa: F401
  12. from tornado.escape import to_unicode, utf8
  13. from tornado import gen
  14. from tornado.httpclient import AsyncHTTPClient
  15. from tornado.httputil import HTTPHeaders, ResponseStartLine
  16. from tornado.ioloop import IOLoop
  17. from tornado.iostream import UnsatisfiableReadError
  18. from tornado.locks import Event
  19. from tornado.log import gen_log
  20. from tornado.netutil import Resolver, bind_sockets
  21. from tornado.simple_httpclient import (
  22. SimpleAsyncHTTPClient,
  23. HTTPStreamClosedError,
  24. HTTPTimeoutError,
  25. )
  26. from tornado.test.httpclient_test import (
  27. ChunkHandler,
  28. CountdownHandler,
  29. HelloWorldHandler,
  30. RedirectHandler,
  31. )
  32. from tornado.test import httpclient_test
  33. from tornado.testing import (
  34. AsyncHTTPTestCase,
  35. AsyncHTTPSTestCase,
  36. AsyncTestCase,
  37. ExpectLog,
  38. gen_test,
  39. )
  40. from tornado.test.util import skipOnTravis, skipIfNoIPv6, refusing_port
  41. from tornado.web import RequestHandler, Application, url, stream_request_body
  42. class SimpleHTTPClientCommonTestCase(httpclient_test.HTTPClientCommonTestCase):
  43. def get_http_client(self):
  44. client = SimpleAsyncHTTPClient(force_instance=True)
  45. self.assertTrue(isinstance(client, SimpleAsyncHTTPClient))
  46. return client
  47. class TriggerHandler(RequestHandler):
  48. def initialize(self, queue, wake_callback):
  49. self.queue = queue
  50. self.wake_callback = wake_callback
  51. @gen.coroutine
  52. def get(self):
  53. logging.debug("queuing trigger")
  54. event = Event()
  55. self.queue.append(event.set)
  56. if self.get_argument("wake", "true") == "true":
  57. self.wake_callback()
  58. yield event.wait()
  59. class ContentLengthHandler(RequestHandler):
  60. def get(self):
  61. self.stream = self.detach()
  62. IOLoop.current().spawn_callback(self.write_response)
  63. @gen.coroutine
  64. def write_response(self):
  65. yield self.stream.write(
  66. utf8(
  67. "HTTP/1.0 200 OK\r\nContent-Length: %s\r\n\r\nok"
  68. % self.get_argument("value")
  69. )
  70. )
  71. self.stream.close()
  72. class HeadHandler(RequestHandler):
  73. def head(self):
  74. self.set_header("Content-Length", "7")
  75. class OptionsHandler(RequestHandler):
  76. def options(self):
  77. self.set_header("Access-Control-Allow-Origin", "*")
  78. self.write("ok")
  79. class NoContentHandler(RequestHandler):
  80. def get(self):
  81. self.set_status(204)
  82. self.finish()
  83. class SeeOtherPostHandler(RequestHandler):
  84. def post(self):
  85. redirect_code = int(self.request.body)
  86. assert redirect_code in (302, 303), "unexpected body %r" % self.request.body
  87. self.set_header("Location", "/see_other_get")
  88. self.set_status(redirect_code)
  89. class SeeOtherGetHandler(RequestHandler):
  90. def get(self):
  91. if self.request.body:
  92. raise Exception("unexpected body %r" % self.request.body)
  93. self.write("ok")
  94. class HostEchoHandler(RequestHandler):
  95. def get(self):
  96. self.write(self.request.headers["Host"])
  97. class NoContentLengthHandler(RequestHandler):
  98. def get(self):
  99. if self.request.version.startswith("HTTP/1"):
  100. # Emulate the old HTTP/1.0 behavior of returning a body with no
  101. # content-length. Tornado handles content-length at the framework
  102. # level so we have to go around it.
  103. stream = self.detach()
  104. stream.write(b"HTTP/1.0 200 OK\r\n\r\n" b"hello")
  105. stream.close()
  106. else:
  107. self.finish("HTTP/1 required")
  108. class EchoPostHandler(RequestHandler):
  109. def post(self):
  110. self.write(self.request.body)
  111. @stream_request_body
  112. class RespondInPrepareHandler(RequestHandler):
  113. def prepare(self):
  114. self.set_status(403)
  115. self.finish("forbidden")
  116. class SimpleHTTPClientTestMixin(object):
  117. def get_app(self):
  118. # callable objects to finish pending /trigger requests
  119. self.triggers = collections.deque() # type: typing.Deque[str]
  120. return Application(
  121. [
  122. url(
  123. "/trigger",
  124. TriggerHandler,
  125. dict(queue=self.triggers, wake_callback=self.stop),
  126. ),
  127. url("/chunk", ChunkHandler),
  128. url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
  129. url("/hello", HelloWorldHandler),
  130. url("/content_length", ContentLengthHandler),
  131. url("/head", HeadHandler),
  132. url("/options", OptionsHandler),
  133. url("/no_content", NoContentHandler),
  134. url("/see_other_post", SeeOtherPostHandler),
  135. url("/see_other_get", SeeOtherGetHandler),
  136. url("/host_echo", HostEchoHandler),
  137. url("/no_content_length", NoContentLengthHandler),
  138. url("/echo_post", EchoPostHandler),
  139. url("/respond_in_prepare", RespondInPrepareHandler),
  140. url("/redirect", RedirectHandler),
  141. ],
  142. gzip=True,
  143. )
  144. def test_singleton(self):
  145. # Class "constructor" reuses objects on the same IOLoop
  146. self.assertTrue(SimpleAsyncHTTPClient() is SimpleAsyncHTTPClient())
  147. # unless force_instance is used
  148. self.assertTrue(
  149. SimpleAsyncHTTPClient() is not SimpleAsyncHTTPClient(force_instance=True)
  150. )
  151. # different IOLoops use different objects
  152. with closing(IOLoop()) as io_loop2:
  153. async def make_client():
  154. await gen.sleep(0)
  155. return SimpleAsyncHTTPClient()
  156. client1 = self.io_loop.run_sync(make_client)
  157. client2 = io_loop2.run_sync(make_client)
  158. self.assertTrue(client1 is not client2)
  159. def test_connection_limit(self):
  160. with closing(self.create_client(max_clients=2)) as client:
  161. self.assertEqual(client.max_clients, 2)
  162. seen = []
  163. # Send 4 requests. Two can be sent immediately, while the others
  164. # will be queued
  165. for i in range(4):
  166. def cb(fut, i=i):
  167. seen.append(i)
  168. self.stop()
  169. client.fetch(self.get_url("/trigger")).add_done_callback(cb)
  170. self.wait(condition=lambda: len(self.triggers) == 2)
  171. self.assertEqual(len(client.queue), 2)
  172. # Finish the first two requests and let the next two through
  173. self.triggers.popleft()()
  174. self.triggers.popleft()()
  175. self.wait(condition=lambda: (len(self.triggers) == 2 and len(seen) == 2))
  176. self.assertEqual(set(seen), set([0, 1]))
  177. self.assertEqual(len(client.queue), 0)
  178. # Finish all the pending requests
  179. self.triggers.popleft()()
  180. self.triggers.popleft()()
  181. self.wait(condition=lambda: len(seen) == 4)
  182. self.assertEqual(set(seen), set([0, 1, 2, 3]))
  183. self.assertEqual(len(self.triggers), 0)
  184. @gen_test
  185. def test_redirect_connection_limit(self):
  186. # following redirects should not consume additional connections
  187. with closing(self.create_client(max_clients=1)) as client:
  188. response = yield client.fetch(self.get_url("/countdown/3"), max_redirects=3)
  189. response.rethrow()
  190. def test_gzip(self):
  191. # All the tests in this file should be using gzip, but this test
  192. # ensures that it is in fact getting compressed.
  193. # Setting Accept-Encoding manually bypasses the client's
  194. # decompression so we can see the raw data.
  195. response = self.fetch(
  196. "/chunk", use_gzip=False, headers={"Accept-Encoding": "gzip"}
  197. )
  198. self.assertEqual(response.headers["Content-Encoding"], "gzip")
  199. self.assertNotEqual(response.body, b"asdfqwer")
  200. # Our test data gets bigger when gzipped. Oops. :)
  201. # Chunked encoding bypasses the MIN_LENGTH check.
  202. self.assertEqual(len(response.body), 34)
  203. f = gzip.GzipFile(mode="r", fileobj=response.buffer)
  204. self.assertEqual(f.read(), b"asdfqwer")
  205. def test_max_redirects(self):
  206. response = self.fetch("/countdown/5", max_redirects=3)
  207. self.assertEqual(302, response.code)
  208. # We requested 5, followed three redirects for 4, 3, 2, then the last
  209. # unfollowed redirect is to 1.
  210. self.assertTrue(response.request.url.endswith("/countdown/5"))
  211. self.assertTrue(response.effective_url.endswith("/countdown/2"))
  212. self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
  213. def test_header_reuse(self):
  214. # Apps may reuse a headers object if they are only passing in constant
  215. # headers like user-agent. The header object should not be modified.
  216. headers = HTTPHeaders({"User-Agent": "Foo"})
  217. self.fetch("/hello", headers=headers)
  218. self.assertEqual(list(headers.get_all()), [("User-Agent", "Foo")])
  219. def test_see_other_redirect(self):
  220. for code in (302, 303):
  221. response = self.fetch("/see_other_post", method="POST", body="%d" % code)
  222. self.assertEqual(200, response.code)
  223. self.assertTrue(response.request.url.endswith("/see_other_post"))
  224. self.assertTrue(response.effective_url.endswith("/see_other_get"))
  225. # request is the original request, is a POST still
  226. self.assertEqual("POST", response.request.method)
  227. @skipOnTravis
  228. @gen_test
  229. def test_connect_timeout(self):
  230. timeout = 0.1
  231. cleanup_event = Event()
  232. test = self
  233. class TimeoutResolver(Resolver):
  234. async def resolve(self, *args, **kwargs):
  235. await cleanup_event.wait()
  236. # Return something valid so the test doesn't raise during shutdown.
  237. return [(socket.AF_INET, ("127.0.0.1", test.get_http_port()))]
  238. with closing(self.create_client(resolver=TimeoutResolver())) as client:
  239. with self.assertRaises(HTTPTimeoutError):
  240. yield client.fetch(
  241. self.get_url("/hello"),
  242. connect_timeout=timeout,
  243. request_timeout=3600,
  244. raise_error=True,
  245. )
  246. # Let the hanging coroutine clean up after itself. We need to
  247. # wait more than a single IOLoop iteration for the SSL case,
  248. # which logs errors on unexpected EOF.
  249. cleanup_event.set()
  250. yield gen.sleep(0.2)
  251. @skipOnTravis
  252. def test_request_timeout(self):
  253. timeout = 0.1
  254. if os.name == "nt":
  255. timeout = 0.5
  256. with self.assertRaises(HTTPTimeoutError):
  257. self.fetch("/trigger?wake=false", request_timeout=timeout, raise_error=True)
  258. # trigger the hanging request to let it clean up after itself
  259. self.triggers.popleft()()
  260. self.io_loop.run_sync(lambda: gen.sleep(0))
  261. @skipIfNoIPv6
  262. def test_ipv6(self):
  263. [sock] = bind_sockets(0, "::1", family=socket.AF_INET6)
  264. port = sock.getsockname()[1]
  265. self.http_server.add_socket(sock)
  266. url = "%s://[::1]:%d/hello" % (self.get_protocol(), port)
  267. # ipv6 is currently enabled by default but can be disabled
  268. with self.assertRaises(Exception):
  269. self.fetch(url, allow_ipv6=False, raise_error=True)
  270. response = self.fetch(url)
  271. self.assertEqual(response.body, b"Hello world!")
  272. def test_multiple_content_length_accepted(self):
  273. response = self.fetch("/content_length?value=2,2")
  274. self.assertEqual(response.body, b"ok")
  275. response = self.fetch("/content_length?value=2,%202,2")
  276. self.assertEqual(response.body, b"ok")
  277. with ExpectLog(gen_log, ".*Multiple unequal Content-Lengths"):
  278. with self.assertRaises(HTTPStreamClosedError):
  279. self.fetch("/content_length?value=2,4", raise_error=True)
  280. with self.assertRaises(HTTPStreamClosedError):
  281. self.fetch("/content_length?value=2,%202,3", raise_error=True)
  282. def test_head_request(self):
  283. response = self.fetch("/head", method="HEAD")
  284. self.assertEqual(response.code, 200)
  285. self.assertEqual(response.headers["content-length"], "7")
  286. self.assertFalse(response.body)
  287. def test_options_request(self):
  288. response = self.fetch("/options", method="OPTIONS")
  289. self.assertEqual(response.code, 200)
  290. self.assertEqual(response.headers["content-length"], "2")
  291. self.assertEqual(response.headers["access-control-allow-origin"], "*")
  292. self.assertEqual(response.body, b"ok")
  293. def test_no_content(self):
  294. response = self.fetch("/no_content")
  295. self.assertEqual(response.code, 204)
  296. # 204 status shouldn't have a content-length
  297. #
  298. # Tests with a content-length header are included below
  299. # in HTTP204NoContentTestCase.
  300. self.assertNotIn("Content-Length", response.headers)
  301. def test_host_header(self):
  302. host_re = re.compile(b"^127.0.0.1:[0-9]+$")
  303. response = self.fetch("/host_echo")
  304. self.assertTrue(host_re.match(response.body))
  305. url = self.get_url("/host_echo").replace("http://", "http://me:secret@")
  306. response = self.fetch(url)
  307. self.assertTrue(host_re.match(response.body), response.body)
  308. def test_connection_refused(self):
  309. cleanup_func, port = refusing_port()
  310. self.addCleanup(cleanup_func)
  311. with ExpectLog(gen_log, ".*", required=False):
  312. with self.assertRaises(socket.error) as cm:
  313. self.fetch("http://127.0.0.1:%d/" % port, raise_error=True)
  314. if sys.platform != "cygwin":
  315. # cygwin returns EPERM instead of ECONNREFUSED here
  316. contains_errno = str(errno.ECONNREFUSED) in str(cm.exception)
  317. if not contains_errno and hasattr(errno, "WSAECONNREFUSED"):
  318. contains_errno = str(errno.WSAECONNREFUSED) in str( # type: ignore
  319. cm.exception
  320. )
  321. self.assertTrue(contains_errno, cm.exception)
  322. # This is usually "Connection refused".
  323. # On windows, strerror is broken and returns "Unknown error".
  324. expected_message = os.strerror(errno.ECONNREFUSED)
  325. self.assertTrue(expected_message in str(cm.exception), cm.exception)
  326. def test_queue_timeout(self):
  327. with closing(self.create_client(max_clients=1)) as client:
  328. # Wait for the trigger request to block, not complete.
  329. fut1 = client.fetch(self.get_url("/trigger"), request_timeout=10)
  330. self.wait()
  331. with self.assertRaises(HTTPTimeoutError) as cm:
  332. self.io_loop.run_sync(
  333. lambda: client.fetch(
  334. self.get_url("/hello"), connect_timeout=0.1, raise_error=True
  335. )
  336. )
  337. self.assertEqual(str(cm.exception), "Timeout in request queue")
  338. self.triggers.popleft()()
  339. self.io_loop.run_sync(lambda: fut1)
  340. def test_no_content_length(self):
  341. response = self.fetch("/no_content_length")
  342. if response.body == b"HTTP/1 required":
  343. self.skipTest("requires HTTP/1.x")
  344. else:
  345. self.assertEquals(b"hello", response.body)
  346. def sync_body_producer(self, write):
  347. write(b"1234")
  348. write(b"5678")
  349. @gen.coroutine
  350. def async_body_producer(self, write):
  351. yield write(b"1234")
  352. yield gen.moment
  353. yield write(b"5678")
  354. def test_sync_body_producer_chunked(self):
  355. response = self.fetch(
  356. "/echo_post", method="POST", body_producer=self.sync_body_producer
  357. )
  358. response.rethrow()
  359. self.assertEqual(response.body, b"12345678")
  360. def test_sync_body_producer_content_length(self):
  361. response = self.fetch(
  362. "/echo_post",
  363. method="POST",
  364. body_producer=self.sync_body_producer,
  365. headers={"Content-Length": "8"},
  366. )
  367. response.rethrow()
  368. self.assertEqual(response.body, b"12345678")
  369. def test_async_body_producer_chunked(self):
  370. response = self.fetch(
  371. "/echo_post", method="POST", body_producer=self.async_body_producer
  372. )
  373. response.rethrow()
  374. self.assertEqual(response.body, b"12345678")
  375. def test_async_body_producer_content_length(self):
  376. response = self.fetch(
  377. "/echo_post",
  378. method="POST",
  379. body_producer=self.async_body_producer,
  380. headers={"Content-Length": "8"},
  381. )
  382. response.rethrow()
  383. self.assertEqual(response.body, b"12345678")
  384. def test_native_body_producer_chunked(self):
  385. async def body_producer(write):
  386. await write(b"1234")
  387. import asyncio
  388. await asyncio.sleep(0)
  389. await write(b"5678")
  390. response = self.fetch("/echo_post", method="POST", body_producer=body_producer)
  391. response.rethrow()
  392. self.assertEqual(response.body, b"12345678")
  393. def test_native_body_producer_content_length(self):
  394. async def body_producer(write):
  395. await write(b"1234")
  396. import asyncio
  397. await asyncio.sleep(0)
  398. await write(b"5678")
  399. response = self.fetch(
  400. "/echo_post",
  401. method="POST",
  402. body_producer=body_producer,
  403. headers={"Content-Length": "8"},
  404. )
  405. response.rethrow()
  406. self.assertEqual(response.body, b"12345678")
  407. def test_100_continue(self):
  408. response = self.fetch(
  409. "/echo_post", method="POST", body=b"1234", expect_100_continue=True
  410. )
  411. self.assertEqual(response.body, b"1234")
  412. def test_100_continue_early_response(self):
  413. def body_producer(write):
  414. raise Exception("should not be called")
  415. response = self.fetch(
  416. "/respond_in_prepare",
  417. method="POST",
  418. body_producer=body_producer,
  419. expect_100_continue=True,
  420. )
  421. self.assertEqual(response.code, 403)
  422. def test_streaming_follow_redirects(self):
  423. # When following redirects, header and streaming callbacks
  424. # should only be called for the final result.
  425. # TODO(bdarnell): this test belongs in httpclient_test instead of
  426. # simple_httpclient_test, but it fails with the version of libcurl
  427. # available on travis-ci. Move it when that has been upgraded
  428. # or we have a better framework to skip tests based on curl version.
  429. headers = [] # type: typing.List[str]
  430. chunk_bytes = [] # type: typing.List[bytes]
  431. self.fetch(
  432. "/redirect?url=/hello",
  433. header_callback=headers.append,
  434. streaming_callback=chunk_bytes.append,
  435. )
  436. chunks = list(map(to_unicode, chunk_bytes))
  437. self.assertEqual(chunks, ["Hello world!"])
  438. # Make sure we only got one set of headers.
  439. num_start_lines = len([h for h in headers if h.startswith("HTTP/")])
  440. self.assertEqual(num_start_lines, 1)
  441. class SimpleHTTPClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPTestCase):
  442. def setUp(self):
  443. super(SimpleHTTPClientTestCase, self).setUp()
  444. self.http_client = self.create_client()
  445. def create_client(self, **kwargs):
  446. return SimpleAsyncHTTPClient(force_instance=True, **kwargs)
  447. class SimpleHTTPSClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPSTestCase):
  448. def setUp(self):
  449. super(SimpleHTTPSClientTestCase, self).setUp()
  450. self.http_client = self.create_client()
  451. def create_client(self, **kwargs):
  452. return SimpleAsyncHTTPClient(
  453. force_instance=True, defaults=dict(validate_cert=False), **kwargs
  454. )
  455. def test_ssl_options(self):
  456. resp = self.fetch("/hello", ssl_options={})
  457. self.assertEqual(resp.body, b"Hello world!")
  458. def test_ssl_context(self):
  459. resp = self.fetch("/hello", ssl_options=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
  460. self.assertEqual(resp.body, b"Hello world!")
  461. def test_ssl_options_handshake_fail(self):
  462. with ExpectLog(gen_log, "SSL Error|Uncaught exception", required=False):
  463. with self.assertRaises(ssl.SSLError):
  464. self.fetch(
  465. "/hello",
  466. ssl_options=dict(cert_reqs=ssl.CERT_REQUIRED),
  467. raise_error=True,
  468. )
  469. def test_ssl_context_handshake_fail(self):
  470. with ExpectLog(gen_log, "SSL Error|Uncaught exception"):
  471. ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
  472. ctx.verify_mode = ssl.CERT_REQUIRED
  473. with self.assertRaises(ssl.SSLError):
  474. self.fetch("/hello", ssl_options=ctx, raise_error=True)
  475. def test_error_logging(self):
  476. # No stack traces are logged for SSL errors (in this case,
  477. # failure to validate the testing self-signed cert).
  478. # The SSLError is exposed through ssl.SSLError.
  479. with ExpectLog(gen_log, ".*") as expect_log:
  480. with self.assertRaises(ssl.SSLError):
  481. self.fetch("/", validate_cert=True, raise_error=True)
  482. self.assertFalse(expect_log.logged_stack)
  483. class CreateAsyncHTTPClientTestCase(AsyncTestCase):
  484. def setUp(self):
  485. super(CreateAsyncHTTPClientTestCase, self).setUp()
  486. self.saved = AsyncHTTPClient._save_configuration()
  487. def tearDown(self):
  488. AsyncHTTPClient._restore_configuration(self.saved)
  489. super(CreateAsyncHTTPClientTestCase, self).tearDown()
  490. def test_max_clients(self):
  491. AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
  492. with closing(AsyncHTTPClient(force_instance=True)) as client:
  493. self.assertEqual(client.max_clients, 10) # type: ignore
  494. with closing(AsyncHTTPClient(max_clients=11, force_instance=True)) as client:
  495. self.assertEqual(client.max_clients, 11) # type: ignore
  496. # Now configure max_clients statically and try overriding it
  497. # with each way max_clients can be passed
  498. AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12)
  499. with closing(AsyncHTTPClient(force_instance=True)) as client:
  500. self.assertEqual(client.max_clients, 12) # type: ignore
  501. with closing(AsyncHTTPClient(max_clients=13, force_instance=True)) as client:
  502. self.assertEqual(client.max_clients, 13) # type: ignore
  503. with closing(AsyncHTTPClient(max_clients=14, force_instance=True)) as client:
  504. self.assertEqual(client.max_clients, 14) # type: ignore
  505. class HTTP100ContinueTestCase(AsyncHTTPTestCase):
  506. def respond_100(self, request):
  507. self.http1 = request.version.startswith("HTTP/1.")
  508. if not self.http1:
  509. request.connection.write_headers(
  510. ResponseStartLine("", 200, "OK"), HTTPHeaders()
  511. )
  512. request.connection.finish()
  513. return
  514. self.request = request
  515. fut = self.request.connection.stream.write(b"HTTP/1.1 100 CONTINUE\r\n\r\n")
  516. fut.add_done_callback(self.respond_200)
  517. def respond_200(self, fut):
  518. fut.result()
  519. fut = self.request.connection.stream.write(
  520. b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\nA"
  521. )
  522. fut.add_done_callback(lambda f: self.request.connection.stream.close())
  523. def get_app(self):
  524. # Not a full Application, but works as an HTTPServer callback
  525. return self.respond_100
  526. def test_100_continue(self):
  527. res = self.fetch("/")
  528. if not self.http1:
  529. self.skipTest("requires HTTP/1.x")
  530. self.assertEqual(res.body, b"A")
  531. class HTTP204NoContentTestCase(AsyncHTTPTestCase):
  532. def respond_204(self, request):
  533. self.http1 = request.version.startswith("HTTP/1.")
  534. if not self.http1:
  535. # Close the request cleanly in HTTP/2; it will be skipped anyway.
  536. request.connection.write_headers(
  537. ResponseStartLine("", 200, "OK"), HTTPHeaders()
  538. )
  539. request.connection.finish()
  540. return
  541. # A 204 response never has a body, even if doesn't have a content-length
  542. # (which would otherwise mean read-until-close). We simulate here a
  543. # server that sends no content length and does not close the connection.
  544. #
  545. # Tests of a 204 response with no Content-Length header are included
  546. # in SimpleHTTPClientTestMixin.
  547. stream = request.connection.detach()
  548. stream.write(b"HTTP/1.1 204 No content\r\n")
  549. if request.arguments.get("error", [False])[-1]:
  550. stream.write(b"Content-Length: 5\r\n")
  551. else:
  552. stream.write(b"Content-Length: 0\r\n")
  553. stream.write(b"\r\n")
  554. stream.close()
  555. def get_app(self):
  556. return self.respond_204
  557. def test_204_no_content(self):
  558. resp = self.fetch("/")
  559. if not self.http1:
  560. self.skipTest("requires HTTP/1.x")
  561. self.assertEqual(resp.code, 204)
  562. self.assertEqual(resp.body, b"")
  563. def test_204_invalid_content_length(self):
  564. # 204 status with non-zero content length is malformed
  565. with ExpectLog(gen_log, ".*Response with code 204 should not have body"):
  566. with self.assertRaises(HTTPStreamClosedError):
  567. self.fetch("/?error=1", raise_error=True)
  568. if not self.http1:
  569. self.skipTest("requires HTTP/1.x")
  570. if self.http_client.configured_class != SimpleAsyncHTTPClient:
  571. self.skipTest("curl client accepts invalid headers")
  572. class HostnameMappingTestCase(AsyncHTTPTestCase):
  573. def setUp(self):
  574. super(HostnameMappingTestCase, self).setUp()
  575. self.http_client = SimpleAsyncHTTPClient(
  576. hostname_mapping={
  577. "www.example.com": "127.0.0.1",
  578. ("foo.example.com", 8000): ("127.0.0.1", self.get_http_port()),
  579. }
  580. )
  581. def get_app(self):
  582. return Application([url("/hello", HelloWorldHandler)])
  583. def test_hostname_mapping(self):
  584. response = self.fetch("http://www.example.com:%d/hello" % self.get_http_port())
  585. response.rethrow()
  586. self.assertEqual(response.body, b"Hello world!")
  587. def test_port_mapping(self):
  588. response = self.fetch("http://foo.example.com:8000/hello")
  589. response.rethrow()
  590. self.assertEqual(response.body, b"Hello world!")
  591. class ResolveTimeoutTestCase(AsyncHTTPTestCase):
  592. def setUp(self):
  593. self.cleanup_event = Event()
  594. test = self
  595. # Dummy Resolver subclass that never finishes.
  596. class BadResolver(Resolver):
  597. @gen.coroutine
  598. def resolve(self, *args, **kwargs):
  599. yield test.cleanup_event.wait()
  600. # Return something valid so the test doesn't raise during cleanup.
  601. return [(socket.AF_INET, ("127.0.0.1", test.get_http_port()))]
  602. super(ResolveTimeoutTestCase, self).setUp()
  603. self.http_client = SimpleAsyncHTTPClient(resolver=BadResolver())
  604. def get_app(self):
  605. return Application([url("/hello", HelloWorldHandler)])
  606. def test_resolve_timeout(self):
  607. with self.assertRaises(HTTPTimeoutError):
  608. self.fetch("/hello", connect_timeout=0.1, raise_error=True)
  609. # Let the hanging coroutine clean up after itself
  610. self.cleanup_event.set()
  611. self.io_loop.run_sync(lambda: gen.sleep(0))
  612. class MaxHeaderSizeTest(AsyncHTTPTestCase):
  613. def get_app(self):
  614. class SmallHeaders(RequestHandler):
  615. def get(self):
  616. self.set_header("X-Filler", "a" * 100)
  617. self.write("ok")
  618. class LargeHeaders(RequestHandler):
  619. def get(self):
  620. self.set_header("X-Filler", "a" * 1000)
  621. self.write("ok")
  622. return Application([("/small", SmallHeaders), ("/large", LargeHeaders)])
  623. def get_http_client(self):
  624. return SimpleAsyncHTTPClient(max_header_size=1024)
  625. def test_small_headers(self):
  626. response = self.fetch("/small")
  627. response.rethrow()
  628. self.assertEqual(response.body, b"ok")
  629. def test_large_headers(self):
  630. with ExpectLog(gen_log, "Unsatisfiable read"):
  631. with self.assertRaises(UnsatisfiableReadError):
  632. self.fetch("/large", raise_error=True)
  633. class MaxBodySizeTest(AsyncHTTPTestCase):
  634. def get_app(self):
  635. class SmallBody(RequestHandler):
  636. def get(self):
  637. self.write("a" * 1024 * 64)
  638. class LargeBody(RequestHandler):
  639. def get(self):
  640. self.write("a" * 1024 * 100)
  641. return Application([("/small", SmallBody), ("/large", LargeBody)])
  642. def get_http_client(self):
  643. return SimpleAsyncHTTPClient(max_body_size=1024 * 64)
  644. def test_small_body(self):
  645. response = self.fetch("/small")
  646. response.rethrow()
  647. self.assertEqual(response.body, b"a" * 1024 * 64)
  648. def test_large_body(self):
  649. with ExpectLog(
  650. gen_log, "Malformed HTTP message from None: Content-Length too long"
  651. ):
  652. with self.assertRaises(HTTPStreamClosedError):
  653. self.fetch("/large", raise_error=True)
  654. class MaxBufferSizeTest(AsyncHTTPTestCase):
  655. def get_app(self):
  656. class LargeBody(RequestHandler):
  657. def get(self):
  658. self.write("a" * 1024 * 100)
  659. return Application([("/large", LargeBody)])
  660. def get_http_client(self):
  661. # 100KB body with 64KB buffer
  662. return SimpleAsyncHTTPClient(
  663. max_body_size=1024 * 100, max_buffer_size=1024 * 64
  664. )
  665. def test_large_body(self):
  666. response = self.fetch("/large")
  667. response.rethrow()
  668. self.assertEqual(response.body, b"a" * 1024 * 100)
  669. class ChunkedWithContentLengthTest(AsyncHTTPTestCase):
  670. def get_app(self):
  671. class ChunkedWithContentLength(RequestHandler):
  672. def get(self):
  673. # Add an invalid Transfer-Encoding to the response
  674. self.set_header("Transfer-Encoding", "chunked")
  675. self.write("Hello world")
  676. return Application([("/chunkwithcl", ChunkedWithContentLength)])
  677. def get_http_client(self):
  678. return SimpleAsyncHTTPClient()
  679. def test_chunked_with_content_length(self):
  680. # Make sure the invalid headers are detected
  681. with ExpectLog(
  682. gen_log,
  683. (
  684. "Malformed HTTP message from None: Response "
  685. "with both Transfer-Encoding and Content-Length"
  686. ),
  687. ):
  688. with self.assertRaises(HTTPStreamClosedError):
  689. self.fetch("/chunkwithcl", raise_error=True)