From 473a469a96ac4b13339bb585fc23cbf4d9c82263 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 7 Nov 2023 20:45:50 -0800 Subject: [PATCH 01/20] WIP Demo of Pyodide support --- aiohttp/client.py | 116 +++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 42 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 2c7dfa005a0..71b696de5d2 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -87,7 +87,7 @@ ) from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter from .http_websocket import WSHandshakeError, WSMessage, ws_ext_gen, ws_ext_parse -from .streams import FlowControlDataQueue +from .streams import FlowControlDataQueue, DataQueue from .tracing import Trace, TraceConfig from .typedefs import JSONEncoder, LooseCookies, LooseHeaders, StrOrURL @@ -507,53 +507,85 @@ async def _request( trust_env=self.trust_env, ) - # connection timeout - try: - async with ceil_timeout( - real_timeout.connect, - ceil_threshold=real_timeout.ceil_threshold, - ): - assert self._connector is not None - conn = await self._connector.connect( - req, traces=traces, timeout=real_timeout - ) - except asyncio.TimeoutError as exc: - raise ServerTimeoutError( - f"Connection timeout to host {url}" - ) from exc - - assert conn.transport is not None - - assert conn.protocol is not None - conn.protocol.set_response_params( - timer=timer, - skip_payload=method_must_be_empty_body(method), - read_until_eof=read_until_eof, - auto_decompress=auto_decompress, - read_timeout=real_timeout.sock_read, - read_bufsize=read_bufsize, - timeout_ceil_threshold=self._connector._timeout_ceil_threshold, - max_line_size=max_line_size, - max_field_size=max_field_size, - ) + if sys.platform == "emscripten": + req.response = resp = req.response_class( + req.method, + req.original_url, + writer=None, + continue100=req._continue, + timer=req._timer, + request_info=req.request_info, + traces=req._traces, + loop=req.loop, + session=req._session, + ) + from js import fetch, Headers, AbortController + from pyodide.ffi import to_js + body = None + if req.body: + body = to_js(req.body._value) + abortcontroller = AbortController.new() + tm.register(abortcontroller.abort) + jsresp = await fetch(str(req.url), method=req.method, headers=Headers.new(headers.items()), body=body, signal=abortcontroller.signal) + resp.version = version + resp.status = jsresp.status + resp.reason = jsresp.statusText + # This is not quite correct in handling of repeated headers + resp._headers = CIMultiDict(jsresp.headers) + resp._raw_headers = tuple(tuple(e) for e in jsresp.headers) + resp.content = DataQueue(self._loop) + def done_callback(fut): + resp.content.feed_data(fut.result()) + resp.content.feed_eof() + jsresp.arrayBuffer().add_done_callback(done_callback) + else: + # connection timeout + try: + async with ceil_timeout( + real_timeout.connect, + ceil_threshold=real_timeout.ceil_threshold, + ): + assert self._connector is not None + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) + except asyncio.TimeoutError as exc: + raise ServerTimeoutError( + f"Connection timeout to host {url}" + ) from exc + + assert conn.transport is not None + + assert conn.protocol is not None + conn.protocol.set_response_params( + timer=timer, + skip_payload=method_must_be_empty_body(method), + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + read_timeout=real_timeout.sock_read, + read_bufsize=read_bufsize, + timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, + ) - try: try: - resp = await req.send(conn) try: - await resp.start(conn) + resp = await req.send(conn) + try: + await resp.start(conn) + except BaseException: + resp.close() + raise except BaseException: - resp.close() + conn.close() raise - except BaseException: - conn.close() - raise - except ClientError: - raise - except OSError as exc: - if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + except ClientError: raise - raise ClientOSError(*exc.args) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientOSError(*exc.args) from exc self._cookie_jar.update_cookies(resp.cookies, resp.url) From 8dc9f5afe5787ffa82c9ba9711f35a57edcaa8d2 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 13:41:12 -0800 Subject: [PATCH 02/20] Move logic to `PyodideConnector`, `PyodideRequest`, and `PyodideResponse` --- .pre-commit-config.yaml | 3 + aiohttp/client.py | 138 +++++++++++++++++---------------------- aiohttp/client_reqrep.py | 80 ++++++++++++++++++++++- aiohttp/connector.py | 66 ++++++++++++++++++- 4 files changed, 207 insertions(+), 80 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 285b11e96db..e4ae262b06c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,9 @@ repos: rev: v1.5.0 hooks: - id: yesqa + additional_dependencies: + - flake8-docstrings==1.6.0 + - flake8-requirements==1.7.8 - repo: https://github.com/PyCQA/isort rev: '5.12.0' hooks: diff --git a/aiohttp/client.py b/aiohttp/client.py index 71b696de5d2..1160fc8dc4c 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -66,6 +66,8 @@ ClientRequest, ClientResponse, Fingerprint, + PyodideClientRequest, + PyodideClientResponse, RequestInfo, ) from .client_ws import ( @@ -73,10 +75,17 @@ ClientWebSocketResponse, ClientWSTimeout, ) -from .connector import BaseConnector, NamedPipeConnector, TCPConnector, UnixConnector +from .connector import ( + BaseConnector, + NamedPipeConnector, + PyodideConnector, + TCPConnector, + UnixConnector, +) from .cookiejar import CookieJar from .helpers import ( _SENTINEL, + IS_PYODIDE, BasicAuth, TimeoutHandle, ceil_timeout, @@ -87,7 +96,7 @@ ) from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter from .http_websocket import WSHandshakeError, WSMessage, ws_ext_gen, ws_ext_parse -from .streams import FlowControlDataQueue, DataQueue +from .streams import FlowControlDataQueue from .tracing import Trace, TraceConfig from .typedefs import JSONEncoder, LooseCookies, LooseHeaders, StrOrURL @@ -207,8 +216,8 @@ def __init__( skip_auto_headers: Optional[Iterable[str]] = None, auth: Optional[BasicAuth] = None, json_serialize: JSONEncoder = json.dumps, - request_class: Type[ClientRequest] = ClientRequest, - response_class: Type[ClientResponse] = ClientResponse, + request_class: Type[ClientRequest] = None, + response_class: Type[ClientResponse] = None, ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse, version: HttpVersion = http.HttpVersion11, cookie_jar: Optional[AbstractCookieJar] = None, @@ -237,7 +246,7 @@ def __init__( loop = asyncio.get_running_loop() if connector is None: - connector = TCPConnector() + connector = PyodideConnector() if IS_PYODIDE else TCPConnector() # Initialize these three attrs before raising any exception, # they are used in __del__ @@ -287,6 +296,11 @@ def __init__( else: self._skip_auto_headers = frozenset() + if request_class is None: + request_class = PyodideClientRequest if IS_PYODIDE else ClientRequest + if response_class is None: + response_class = PyodideClientResponse if IS_PYODIDE else ClientResponse + self._request_class = request_class self._response_class = response_class self._ws_response_class = ws_response_class @@ -507,85 +521,53 @@ async def _request( trust_env=self.trust_env, ) - if sys.platform == "emscripten": - req.response = resp = req.response_class( - req.method, - req.original_url, - writer=None, - continue100=req._continue, - timer=req._timer, - request_info=req.request_info, - traces=req._traces, - loop=req.loop, - session=req._session, - ) - from js import fetch, Headers, AbortController - from pyodide.ffi import to_js - body = None - if req.body: - body = to_js(req.body._value) - abortcontroller = AbortController.new() - tm.register(abortcontroller.abort) - jsresp = await fetch(str(req.url), method=req.method, headers=Headers.new(headers.items()), body=body, signal=abortcontroller.signal) - resp.version = version - resp.status = jsresp.status - resp.reason = jsresp.statusText - # This is not quite correct in handling of repeated headers - resp._headers = CIMultiDict(jsresp.headers) - resp._raw_headers = tuple(tuple(e) for e in jsresp.headers) - resp.content = DataQueue(self._loop) - def done_callback(fut): - resp.content.feed_data(fut.result()) - resp.content.feed_eof() - jsresp.arrayBuffer().add_done_callback(done_callback) - else: - # connection timeout - try: - async with ceil_timeout( - real_timeout.connect, - ceil_threshold=real_timeout.ceil_threshold, - ): - assert self._connector is not None - conn = await self._connector.connect( - req, traces=traces, timeout=real_timeout - ) - except asyncio.TimeoutError as exc: - raise ServerTimeoutError( - f"Connection timeout to host {url}" - ) from exc - - assert conn.transport is not None - - assert conn.protocol is not None - conn.protocol.set_response_params( - timer=timer, - skip_payload=method_must_be_empty_body(method), - read_until_eof=read_until_eof, - auto_decompress=auto_decompress, - read_timeout=real_timeout.sock_read, - read_bufsize=read_bufsize, - timeout_ceil_threshold=self._connector._timeout_ceil_threshold, - max_line_size=max_line_size, - max_field_size=max_field_size, - ) + # connection timeout + try: + async with ceil_timeout( + real_timeout.connect, + ceil_threshold=real_timeout.ceil_threshold, + ): + assert self._connector is not None + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) + except asyncio.TimeoutError as exc: + raise ServerTimeoutError( + f"Connection timeout to host {url}" + ) from exc + + assert conn.transport is not None + + assert conn.protocol is not None + conn.protocol.set_response_params( + timer=timer, + skip_payload=method_must_be_empty_body(method), + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + read_timeout=real_timeout.sock_read, + read_bufsize=read_bufsize, + timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, + ) + try: try: + resp = await req.send(conn) try: - resp = await req.send(conn) - try: - await resp.start(conn) - except BaseException: - resp.close() - raise + await resp.start(conn) except BaseException: - conn.close() + resp.close() raise - except ClientError: + except BaseException: + conn.close() raise - except OSError as exc: - if exc.errno is None and isinstance(exc, asyncio.TimeoutError): - raise - raise ClientOSError(*exc.args) from exc + except ClientError: + raise + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientOSError(*exc.args) from exc self._cookie_jar.update_cookies(resp.cookies, resp.url) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index f05a08637f2..c5d7c5a41a7 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -44,6 +44,7 @@ from .formdata import FormData from .hdrs import CONTENT_TYPE from .helpers import ( + IS_PYODIDE, BaseTimerContext, BasicAuth, HeadersMixin, @@ -585,7 +586,7 @@ async def write_bytes( await writer.write_eof() protocol.start_timeout() - async def send(self, conn: "Connection") -> "ClientResponse": + def _path(self) -> str: # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI @@ -602,7 +603,10 @@ async def send(self, conn: "Connection") -> "ClientResponse": path = self.url.raw_path if self.url.raw_query_string: path += "?" + self.url.raw_query_string + return path + async def send(self, conn: "Connection") -> "ClientResponse": + path = self._path() protocol = conn.protocol assert protocol is not None writer = StreamWriter( @@ -689,6 +693,52 @@ async def _on_headers_request_sent( await trace.send_request_headers(method, url, headers) +class PyodideClientRequest(ClientRequest): + async def send(self, conn: "Connection") -> "ClientResponse": + if not IS_PYODIDE: + raise RuntimeError("PyodideClientRequest only works in Pyodide") + + path = self._path() + protocol = conn.protocol + assert protocol is not None + from js import Headers, fetch # noqa: I900 + from pyodide.ffi import to_js # noqa: I900 + + body = None + if self.body: + if isinstance(self.body, payload.Payload): + body = to_js(self.body._value) + else: + if isinstance(self.body, (bytes, bytearray)): + body = (self.body,) # type: ignore[assignment] + + raise NotImplementedError("OOPS") + response_future = fetch( + path, + method=self.method, + headers=Headers.new(self.headers.items()), + body=body, + signal=protocol.abortcontroller.signal, + ) + response_class = self.response_class + assert response_class is not None + assert issubclass(response_class, PyodideClientResponse) + self.response = response_class( + self.method, + self.original_url, + writer=None, + continue100=self._continue, + timer=self._timer, + request_info=self.request_info, + traces=self._traces, + loop=self.loop, + session=self._session, + response_future=response_future, + ) + self.response.version = self.version + return self.response + + class ClientResponse(HeadersMixin): # Some of these attributes are None when created, # but will be set by the start() method. @@ -1124,3 +1174,31 @@ async def __aexit__( # if state is broken self.release() await self.wait_for_close() + + +class PyodideClientResponse(ClientResponse): + def __init__(self, *args, response_future, **kwargs): + if not IS_PYODIDE: + raise RuntimeError("PyodideClientResponse only works in Pyodide") + self.response_future = response_future + super().__init__(*args, **kwargs) + + async def start(self, connection: "Connection") -> "ClientResponse": + from .streams import DataQueue + + self._connection = connection + self._protocol = connection.protocol + jsresp = await self.response_future + self.status = jsresp.status + self.reason = jsresp.statusText + # This is not quite correct in handling of repeated headers + self._headers = CIMultiDict(jsresp.headers) + self._raw_headers = tuple(tuple(e) for e in jsresp.headers) + self.content = DataQueue(self._loop) + + def done_callback(fut): + self.content.feed_data(fut.result()) + self.content.feed_eof() + + jsresp.arrayBuffer().add_done_callback(done_callback) + return self diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 2460ca46705..2505e032f9f 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -47,7 +47,14 @@ ) from .client_proto import ResponseHandler from .client_reqrep import SSL_ALLOWED_TYPES, ClientRequest, Fingerprint -from .helpers import _SENTINEL, ceil_timeout, is_ip_address, sentinel, set_result +from .helpers import ( + _SENTINEL, + IS_PYODIDE, + ceil_timeout, + is_ip_address, + sentinel, + set_result, +) from .locks import EventResultOrError from .resolver import DefaultResolver @@ -1384,3 +1391,60 @@ async def _create_connection( raise ClientConnectorError(req.connection_key, exc) from exc return cast(ResponseHandler, proto) + + +IN_PYODIDE = "pyodide" in sys.modules or "emscripten" in sys.platform + + +class PyodideProtocol(ResponseHandler): + def __init__(self): + from js import AbortController # noqa: I900 + + self.abortcontroller = AbortController() + + def close(self): + self.abortcontroller.abort() + + +class PyodideConnector(BaseConnector): + """Named pipe connector. + + Only supported by the proactor event loop. + See also: https://docs.python.org/3/library/asyncio-eventloop.html + + path - Windows named pipe path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: Union[_SENTINEL, float, None] = sentinel, + limit: int = 100, + limit_per_host: int = 0, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + ) + if IS_PYODIDE: + raise RuntimeError("PyodideConnector only works in Pyodide") + self._path = path + + @property + def path(self) -> str: + """Path to the named pipe.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + return PyodideProtocol() From 402880b323c3afd37743122f73dd8b2646ad3a2d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 13:54:51 -0800 Subject: [PATCH 03/20] Add IS_PYODIDE to helpers.py --- aiohttp/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 5435e2f9e07..f0ec213110f 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -108,6 +108,8 @@ } TOKEN = CHAR ^ CTL ^ SEPARATORS +IS_PYODIDE = "pyodide" in sys.modules + class noop: def __await__(self) -> Generator[None, None, None]: From ee5c0f1e31f689734319b8dde013488ee64e2323 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 13:58:21 -0800 Subject: [PATCH 04/20] Fix PyodideConnector constructor --- aiohttp/connector.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 2505e032f9f..b7cec3c982d 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1423,21 +1423,24 @@ class PyodideConnector(BaseConnector): def __init__( self, - path: str, + *, + keepalive_timeout: Union[_SENTINEL, None, float] = sentinel, force_close: bool = False, - keepalive_timeout: Union[_SENTINEL, float, None] = sentinel, limit: int = 100, limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + timeout_ceil_threshold: float = 5, ) -> None: super().__init__( - force_close=force_close, - keepalive_timeout=keepalive_timeout, - limit=limit, - limit_per_host=limit_per_host, + keepalive_timeout, + force_close, + limit, + limit_per_host, + enable_cleanup_closed, + timeout_ceil_threshold, ) if IS_PYODIDE: raise RuntimeError("PyodideConnector only works in Pyodide") - self._path = path @property def path(self) -> str: From 7a4c9db722561ecc1a4ef26965089355bb805207 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 14:00:33 -0800 Subject: [PATCH 05/20] Fix connector constructor again --- aiohttp/connector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index b7cec3c982d..d763015a1f1 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1432,12 +1432,12 @@ def __init__( timeout_ceil_threshold: float = 5, ) -> None: super().__init__( - keepalive_timeout, - force_close, - limit, - limit_per_host, - enable_cleanup_closed, - timeout_ceil_threshold, + keepalive_timeout=keepalive_timeout, + force_close=force_close, + limit=limit, + limit_per_host=limit_per_host, + enable_cleanup_closed=enable_cleanup_closed, + timeout_ceil_threshold=timeout_ceil_threshold, ) if IS_PYODIDE: raise RuntimeError("PyodideConnector only works in Pyodide") From 5572cb03ca99d87a68e06a05cd859fd1b0f0e956 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 14:02:54 -0800 Subject: [PATCH 06/20] Fix negated conditional --- aiohttp/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index d763015a1f1..f0ac4459132 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1439,7 +1439,7 @@ def __init__( enable_cleanup_closed=enable_cleanup_closed, timeout_ceil_threshold=timeout_ceil_threshold, ) - if IS_PYODIDE: + if not IS_PYODIDE: raise RuntimeError("PyodideConnector only works in Pyodide") @property From cb54962a667a4b54b9ae3b3f18bdb328a2e6e19f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 14:27:21 -0800 Subject: [PATCH 07/20] Fix more things --- aiohttp/client_reqrep.py | 3 ++- aiohttp/connector.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index c5d7c5a41a7..01819b60222 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -1197,7 +1197,8 @@ async def start(self, connection: "Connection") -> "ClientResponse": self.content = DataQueue(self._loop) def done_callback(fut): - self.content.feed_data(fut.result()) + data = fut.result().to_bytes() + self.content.feed_data(data, len(data)) self.content.feed_eof() jsresp.arrayBuffer().add_done_callback(done_callback) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index f0ac4459132..fe26a361b70 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1397,13 +1397,18 @@ async def _create_connection( class PyodideProtocol(ResponseHandler): - def __init__(self): + def __init__(self, loop: asyncio.AbstractEventLoop): from js import AbortController # noqa: I900 - self.abortcontroller = AbortController() + super().__init__(loop) + self.abortcontroller = AbortController.new() + self.closed = loop.create_future() + # asyncio.Transport "raises NotImplemented for every method" + self.transport = asyncio.Transport() def close(self): self.abortcontroller.abort() + self.closed.set_result(None) class PyodideConnector(BaseConnector): @@ -1450,4 +1455,4 @@ def path(self) -> str: async def _create_connection( self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" ) -> ResponseHandler: - return PyodideProtocol() + return PyodideProtocol(self._loop) From 72abf4f537f4da799bfaf420d3b99345138e4aa7 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 14:42:30 -0800 Subject: [PATCH 08/20] Try adding Pyodide to CI --- .github/workflows/ci-cd.yml | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index e961250bf67..b63217498a6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -269,6 +269,82 @@ jobs: steps.python-install.outputs.python-version }} + test: + permissions: + contents: read # to fetch code (actions/checkout) + + name: Test + needs: gen_llhttp + runs-on: ubuntu-22.04 + env: + PYODIDE_VERSION: 0.25.0a1 + # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. + # The appropriate versions can be found in the Pyodide repodata.json + # "info" field, or in Makefile.envs: + # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 + PYTHON_VERSION: 3.11.2 + EMSCRIPTEN_VERSION: 3.1.32 + NODE_VERSION: 18 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python ${{ matrix.pyver }} + id: python-install + uses: actions/setup-python@v4 + with: + allow-prereleases: true + python-version: ${{ matrix.pyver }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" # - name: Cache + - name: Cache PyPI + uses: actions/cache@v3.3.2 + with: + key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} + path: ${{ steps.pip-cache.outputs.dir }} + restore-keys: | + pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Install dependencies + run: | + python -m pip install -r requirements/test.in -c requirements/test.txt + - name: Restore llhttp generated files + if: ${{ matrix.no-extensions == '' }} + uses: actions/download-artifact@v3 + with: + name: llhttp + path: vendor/llhttp/build/ + - name: Cythonize + if: ${{ matrix.no-extensions == '' }} + run: | + make cythonize + - uses: mymindstorm/setup-emsdk@ab889da2abbcbb280f91ec4c215d3bb4f3a8f775 # v12 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + - name: Install pyodide-build + run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION + - name: Build + run: | + CFLAGS=-g2 LDFLAGS=-g2 pyodide build + + # - name: set up node + # uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + # with: + # node-version: ${{ env.NODE_VERSION }} + + # - name: Set up Pyodide virtual environment + # run: | + # pyodide venv .venv-pyodide + # source .venv-pyodide/bin/activate + # pip install dist/*.whl + # python -c "import sys; print(sys.platform)" + check: # This job does nothing and is only used for the branch protection if: always() From ac0c8c3afdce2869de5be8c5fc896a1214d91417 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 14:44:23 -0800 Subject: [PATCH 09/20] Fix --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b63217498a6..2b11babaad9 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -269,7 +269,7 @@ jobs: steps.python-install.outputs.python-version }} - test: + test-pyodide: permissions: contents: read # to fetch code (actions/checkout) From a6f39fa1777afaf88f449c8173af2b7496e2df0c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 Nov 2023 18:03:04 -0800 Subject: [PATCH 10/20] Use correct Emscripten version --- .github/workflows/ci-cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2b11babaad9..17e0ac626f0 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -282,8 +282,8 @@ jobs: # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 - PYTHON_VERSION: 3.11.2 - EMSCRIPTEN_VERSION: 3.1.32 + PYTHON_VERSION: 3.11.4 + EMSCRIPTEN_VERSION: 3.1.45 NODE_VERSION: 18 steps: - name: Checkout From 58ec5cc2a77b608b2fac2c178b2eae57c65d428c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 21 Nov 2023 11:36:03 -0800 Subject: [PATCH 11/20] Fix mypy --- .github/workflows/ci-cd.yml | 6 +-- aiohttp/client.py | 4 +- aiohttp/client_reqrep.py | 94 ++++++++++++++++++++++++++++--------- aiohttp/connector.py | 53 +++++++++++++-------- aiohttp/helpers.py | 18 +++++++ 5 files changed, 127 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 17e0ac626f0..0b7129da3f8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -273,7 +273,7 @@ jobs: permissions: contents: read # to fetch code (actions/checkout) - name: Test + name: Test pyodide needs: gen_llhttp runs-on: ubuntu-22.04 env: @@ -303,10 +303,10 @@ jobs: - name: Cache PyPI uses: actions/cache@v3.3.2 with: - key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} + key: pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | - pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- + pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}- - name: Update pip, wheel, setuptools, build, twine run: | python -m pip install -U pip wheel setuptools build twine diff --git a/aiohttp/client.py b/aiohttp/client.py index 1160fc8dc4c..08608e0f7e1 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -216,8 +216,8 @@ def __init__( skip_auto_headers: Optional[Iterable[str]] = None, auth: Optional[BasicAuth] = None, json_serialize: JSONEncoder = json.dumps, - request_class: Type[ClientRequest] = None, - response_class: Type[ClientResponse] = None, + request_class: Optional[Type[ClientRequest]] = None, + response_class: Optional[Type[ClientResponse]] = None, ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse, version: HttpVersion = http.HttpVersion11, cookie_jar: Optional[AbstractCookieJar] = None, diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 01819b60222..fc13dc7d3fe 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -48,6 +48,9 @@ BaseTimerContext, BasicAuth, HeadersMixin, + JsArrayBuffer, + JsRequest, + JsResponse, TimerNoop, basicauth_from_netrc, is_expected_content_type, @@ -87,7 +90,7 @@ if TYPE_CHECKING: # pragma: no cover from .client import ClientSession - from .connector import Connection + from .connector import Connection, PyodideConnection from .tracing import Trace @@ -693,40 +696,60 @@ async def _on_headers_request_sent( await trace.send_request_headers(method, url, headers) +def _make_js_request( + path: str, *, method: str, headers: CIMultiDict[str], signal: Any, body: Any +) -> JsRequest: + from js import Headers, Request # type:ignore[import-not-found] # noqa: I900 + from pyodide.ffi import to_js # type:ignore[import-not-found] # noqa: I900 + + # TODO: to_js does an unnecessary copy. + if isinstance(body, payload.Payload): + body = to_js(body._value) + elif isinstance(body, (bytes, bytearray)): + body = to_js(body) + else: + # What else can happen here? Maybe body could be a list of + # bytes? In that case we should turn it into a Blob. + raise NotImplementedError("OOPS") + + return cast( + JsRequest, + Request.new( + path, + method=method, + headers=Headers.new(headers.items()), + body=body, + signal=signal, + ), + ) + + class PyodideClientRequest(ClientRequest): - async def send(self, conn: "Connection") -> "ClientResponse": + async def send( + self, conn: "PyodideConnection" # type:ignore[override] + ) -> "ClientResponse": if not IS_PYODIDE: raise RuntimeError("PyodideClientRequest only works in Pyodide") path = self._path() protocol = conn.protocol assert protocol is not None - from js import Headers, fetch # noqa: I900 - from pyodide.ffi import to_js # noqa: I900 - body = None - if self.body: - if isinstance(self.body, payload.Payload): - body = to_js(self.body._value) - else: - if isinstance(self.body, (bytes, bytearray)): - body = (self.body,) # type: ignore[assignment] - - raise NotImplementedError("OOPS") - response_future = fetch( + request = _make_js_request( path, method=self.method, - headers=Headers.new(self.headers.items()), - body=body, + headers=self.headers, + body=self.body, signal=protocol.abortcontroller.signal, ) + response_future = protocol.fetch_handler(request) response_class = self.response_class assert response_class is not None assert issubclass(response_class, PyodideClientResponse) self.response = response_class( self.method, self.original_url, - writer=None, + writer=None, # type:ignore[arg-type] continue100=self._continue, timer=self._timer, request_info=self.request_info, @@ -1177,11 +1200,34 @@ async def __aexit__( class PyodideClientResponse(ClientResponse): - def __init__(self, *args, response_future, **kwargs): + def __init__( + self, + method: str, + url: URL, + *, + writer: "asyncio.Task[None]", + continue100: Optional["asyncio.Future[bool]"], + timer: Optional[BaseTimerContext], + request_info: RequestInfo, + traces: List["Trace"], + loop: asyncio.AbstractEventLoop, + session: "ClientSession", + response_future: asyncio.Future[JsResponse], + ): if not IS_PYODIDE: raise RuntimeError("PyodideClientResponse only works in Pyodide") self.response_future = response_future - super().__init__(*args, **kwargs) + super().__init__( + method, + url, + writer=writer, + continue100=continue100, + timer=timer, + request_info=request_info, + traces=traces, + loop=loop, + session=session, + ) async def start(self, connection: "Connection") -> "ClientResponse": from .streams import DataQueue @@ -1192,11 +1238,13 @@ async def start(self, connection: "Connection") -> "ClientResponse": self.status = jsresp.status self.reason = jsresp.statusText # This is not quite correct in handling of repeated headers - self._headers = CIMultiDict(jsresp.headers) - self._raw_headers = tuple(tuple(e) for e in jsresp.headers) - self.content = DataQueue(self._loop) + self._headers = cast(CIMultiDictProxy[str], CIMultiDict(jsresp.headers)) + self._raw_headers = tuple( + (e[0].encode(), e[1].encode()) for e in jsresp.headers + ) + self.content = DataQueue(self._loop) # type:ignore[assignment] - def done_callback(fut): + def done_callback(fut: asyncio.Future[JsArrayBuffer]) -> None: data = fut.result().to_bytes() self.content.feed_data(data, len(data)) self.content.feed_eof() diff --git a/aiohttp/connector.py b/aiohttp/connector.py index fe26a361b70..c4b940d1900 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -24,6 +24,7 @@ List, Literal, Optional, + Protocol, Set, Tuple, Type, @@ -50,6 +51,8 @@ from .helpers import ( _SENTINEL, IS_PYODIDE, + JsRequest, + JsResponse, ceil_timeout, is_ip_address, sentinel, @@ -1397,38 +1400,47 @@ async def _create_connection( class PyodideProtocol(ResponseHandler): - def __init__(self, loop: asyncio.AbstractEventLoop): - from js import AbortController # noqa: I900 + def __init__( + self, + loop: asyncio.AbstractEventLoop, + fetch_handler: Callable[[JsRequest], asyncio.Future[JsResponse]], + ): + from js import AbortController # type:ignore[import-not-found] # noqa: I900 super().__init__(loop) self.abortcontroller = AbortController.new() self.closed = loop.create_future() # asyncio.Transport "raises NotImplemented for every method" self.transport = asyncio.Transport() + self.fetch_handler = fetch_handler - def close(self): + def close(self) -> None: self.abortcontroller.abort() self.closed.set_result(None) -class PyodideConnector(BaseConnector): - """Named pipe connector. +class PyodideConnection(Connection): + _protocol: PyodideProtocol - Only supported by the proactor event loop. - See also: https://docs.python.org/3/library/asyncio-eventloop.html + @property + def protocol(self) -> Optional[PyodideProtocol]: + return self._protocol - path - Windows named pipe path. - keepalive_timeout - (optional) Keep-alive timeout. - force_close - Set to True to force close and do reconnect - after each request (and between redirects). - limit - The total number of simultaneous connections. - limit_per_host - Number of simultaneous connections to one host. - loop - Optional event loop. + +class PyodideConnector(BaseConnector): + """Pyodide connector + + Dummy connector that """ + protocol: PyodideProtocol + def __init__( self, *, + fetch_handler: Optional[ + Callable[[JsRequest], asyncio.Future[JsResponse]] + ] = None, keepalive_timeout: Union[_SENTINEL, None, float] = sentinel, force_close: bool = False, limit: int = 100, @@ -1444,15 +1456,16 @@ def __init__( enable_cleanup_closed=enable_cleanup_closed, timeout_ceil_threshold=timeout_ceil_threshold, ) + if fetch_handler is None: + from js import fetch # noqa: I900 + + fetch_handler = fetch + + self.fetch_handler = fetch_handler if not IS_PYODIDE: raise RuntimeError("PyodideConnector only works in Pyodide") - @property - def path(self) -> str: - """Path to the named pipe.""" - return self._path - async def _create_connection( self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" ) -> ResponseHandler: - return PyodideProtocol(self._loop) + return PyodideProtocol(self._loop, self.fetch_handler) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index f0ec213110f..a0867e988c5 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -1092,3 +1092,21 @@ def should_remove_content_length(method: str, code: int) -> bool: or 100 <= code < 200 or (200 <= code < 300 and method.upper() == hdrs.METH_CONNECT) ) + + +class JsRequest(Protocol): + pass + + +class JsArrayBuffer(Protocol): + def to_bytes(self) -> bytes: + ... + + +class JsResponse(Protocol): + status: int + statusText: str + headers: List[Tuple[str, str]] + + def arrayBuffer(self) -> asyncio.Future[JsArrayBuffer]: + ... From b56f520aaa8463e13d17c81c0deef5c4e94e309f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 21 Nov 2023 12:49:51 -0800 Subject: [PATCH 12/20] Fix some types for old Python versions --- aiohttp/client_reqrep.py | 4 ++-- aiohttp/connector.py | 4 ++-- aiohttp/helpers.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index fc13dc7d3fe..1d9550852c3 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -1212,7 +1212,7 @@ def __init__( traces: List["Trace"], loop: asyncio.AbstractEventLoop, session: "ClientSession", - response_future: asyncio.Future[JsResponse], + response_future: "asyncio.Future[JsResponse]", ): if not IS_PYODIDE: raise RuntimeError("PyodideClientResponse only works in Pyodide") @@ -1244,7 +1244,7 @@ async def start(self, connection: "Connection") -> "ClientResponse": ) self.content = DataQueue(self._loop) # type:ignore[assignment] - def done_callback(fut: asyncio.Future[JsArrayBuffer]) -> None: + def done_callback(fut: "asyncio.Future[JsArrayBuffer]") -> None: data = fut.result().to_bytes() self.content.feed_data(data, len(data)) self.content.feed_eof() diff --git a/aiohttp/connector.py b/aiohttp/connector.py index c4b940d1900..d732d38e5c7 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1403,7 +1403,7 @@ class PyodideProtocol(ResponseHandler): def __init__( self, loop: asyncio.AbstractEventLoop, - fetch_handler: Callable[[JsRequest], asyncio.Future[JsResponse]], + fetch_handler: Callable[[JsRequest], "asyncio.Future[JsResponse]"], ): from js import AbortController # type:ignore[import-not-found] # noqa: I900 @@ -1439,7 +1439,7 @@ def __init__( self, *, fetch_handler: Optional[ - Callable[[JsRequest], asyncio.Future[JsResponse]] + Callable[[JsRequest], "asyncio.Future[JsResponse]"] ] = None, keepalive_timeout: Union[_SENTINEL, None, float] = sentinel, force_close: bool = False, diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index a0867e988c5..a3a4834a205 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -1108,5 +1108,5 @@ class JsResponse(Protocol): statusText: str headers: List[Tuple[str, str]] - def arrayBuffer(self) -> asyncio.Future[JsArrayBuffer]: + def arrayBuffer(self) -> "asyncio.Future[JsArrayBuffer]": ... From f048898d5205073695aefe96358ec13c817f8847 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 4 Dec 2023 14:44:14 -0800 Subject: [PATCH 13/20] Use correct Pythonv version --- .github/workflows/ci-cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 0b7129da3f8..1694660e6fd 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -290,12 +290,12 @@ jobs: uses: actions/checkout@v4 with: submodules: true - - name: Setup Python ${{ matrix.pyver }} + - name: Setup Python ${{ env.PYTHON_VERSION }} id: python-install uses: actions/setup-python@v4 with: allow-prereleases: true - python-version: ${{ matrix.pyver }} + python-version: ${{ env.PYTHON_VERSION }} - name: Get pip cache dir id: pip-cache run: | From 6a31a222534e7593b103aecdd4b81d0b79bd00dc Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 4 Dec 2023 14:58:07 -0800 Subject: [PATCH 14/20] Try setting up virtualenv --- .github/workflows/ci-cd.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 1694660e6fd..81ffbc87a7a 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -323,7 +323,7 @@ jobs: if: ${{ matrix.no-extensions == '' }} run: | make cythonize - - uses: mymindstorm/setup-emsdk@ab889da2abbcbb280f91ec4c215d3bb4f3a8f775 # v12 + - uses: mymindstorm/setup-emsdk@v12 with: version: ${{ env.EMSCRIPTEN_VERSION }} actions-cache-folder: emsdk-cache @@ -333,17 +333,17 @@ jobs: run: | CFLAGS=-g2 LDFLAGS=-g2 pyodide build - # - name: set up node - # uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 - # with: - # node-version: ${{ env.NODE_VERSION }} + - name: set up node + uses: actions/setup-node@v3.8.1 + with: + node-version: ${{ env.NODE_VERSION }} - # - name: Set up Pyodide virtual environment - # run: | - # pyodide venv .venv-pyodide - # source .venv-pyodide/bin/activate - # pip install dist/*.whl - # python -c "import sys; print(sys.platform)" + - name: Set up Pyodide virtual environment + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + pip install dist/*.whl + python -c "import sys; print(sys.platform)" check: # This job does nothing and is only used for the branch protection if: always() From 2cbae1fd16a58d5e0dc04e56701bc37f65937bad Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 5 Dec 2023 12:28:49 -0800 Subject: [PATCH 15/20] Add mock test --- .github/workflows/ci-cd.yml | 6 +- aiohttp/client_reqrep.py | 8 +-- setup.cfg | 3 +- tests/test_pyodide.py | 106 ++++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 tests/test_pyodide.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 81ffbc87a7a..b91362b1908 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -343,7 +343,11 @@ jobs: pyodide venv .venv-pyodide source .venv-pyodide/bin/activate pip install dist/*.whl - python -c "import sys; print(sys.platform)" + + - name: Test + run: | + source .venv-pyodide/bin/activate + check: # This job does nothing and is only used for the branch protection if: always() diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index b913f416495..a6617e6e5c4 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -589,7 +589,7 @@ async def write_bytes( await writer.write_eof() protocol.start_timeout() - def _path(self) -> str: + async def send(self, conn: "Connection") -> "ClientResponse": # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI @@ -606,10 +606,7 @@ def _path(self) -> str: path = self.url.raw_path if self.url.raw_query_string: path += "?" + self.url.raw_query_string - return path - async def send(self, conn: "Connection") -> "ClientResponse": - path = self._path() protocol = conn.protocol assert protocol is not None writer = StreamWriter( @@ -731,12 +728,11 @@ async def send( if not IS_PYODIDE: raise RuntimeError("PyodideClientRequest only works in Pyodide") - path = self._path() protocol = conn.protocol assert protocol is not None request = _make_js_request( - path, + str(self.url), method=self.method, headers=self.headers, body=self.body, diff --git a/setup.cfg b/setup.cfg index 13efc0b7796..600431a3714 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,8 +126,7 @@ addopts = --showlocals # `pytest-cov`: - --cov=aiohttp - --cov=tests/ + # run tests that are not marked with dev_mode -m "not dev_mode" diff --git a/tests/test_pyodide.py b/tests/test_pyodide.py new file mode 100644 index 00000000000..8d7e62db107 --- /dev/null +++ b/tests/test_pyodide.py @@ -0,0 +1,106 @@ +import sys +from asyncio import Future +from collections.abc import Mapping +from types import ModuleType +from typing import Any + +from pytest import fixture + +from aiohttp import ClientSession, client, client_reqrep, connector +from aiohttp.connector import PyodideConnector + + +class JsAbortController: + @staticmethod + def new() -> "JsAbortController": + return JsAbortController() + + def abort(self) -> None: + pass + + @property + def signal(self) -> str: + return "AbortSignal" + + +class JsHeaders: + def __init__(self, items: Mapping): + self.items = dict(items) + + @staticmethod + def new(items: Mapping) -> "JsHeaders": + return JsHeaders(items) + + +class JsRequest: + def __init__(self, path: str, **kwargs: Any): + self.path = path + self.kwargs = kwargs + + @staticmethod + def new(path, **kwargs) -> "JsRequest": + return JsRequest(path, **kwargs) + + +class JsBuffer: + def __init__(self, content: bytes): + self.content = content + + def to_bytes(self) -> bytes: + return self.content + + +class JsResponse: + def __init__( + self, status: int, statusText: str, headers: JsHeaders, body: bytes | None + ): + self.status = status + self.statusText = statusText + self.headers = headers + self.body = body + + def arrayBuffer(self): + fut = Future() + fut.set_result(self.body) + return fut + + +@fixture +def mock_pyodide_env(monkeypatch: Any): + monkeypatch.setattr(client, "IS_PYODIDE", True) + monkeypatch.setattr(connector, "IS_PYODIDE", True) + monkeypatch.setattr(client_reqrep, "IS_PYODIDE", True) + jsmod = ModuleType("js") + jsmod.AbortController = JsAbortController + jsmod.Headers = JsHeaders + jsmod.Request = JsRequest + + monkeypatch.setitem(sys.modules, "js", jsmod) + + +async def test_pyodide_mock(mock_pyodide_env: Any) -> None: + def fetch_handler(request: JsRequest) -> Future[JsResponse]: + assert request.path == "http://example.com" + assert request.kwargs["method"] == "GET" + assert request.kwargs["headers"].items == { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Host": "example.com", + "User-Agent": "Python/3.11 aiohttp/4.0.0a2.dev0", + } + assert request.kwargs["signal"] == "AbortSignal" + assert request.kwargs["body"] == b"" + fut = Future() + resp = JsResponse( + 200, "OK", [["Content-type", "text/html; charset=utf-8"]], JsBuffer(b"abc") + ) + fut.set_result(resp) + return fut + + c = PyodideConnector(fetch_handler=fetch_handler) + async with ClientSession(connector=c) as session: + async with session.get("http://example.com") as response: + assert response.status == 200 + assert response.headers["content-type"] == "text/html; charset=utf-8" + html = await response.text() + assert html == "abc" From b0208a3f5b8574f4173f196e647f5093fd3c4a2e Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 5 Dec 2023 13:06:52 -0800 Subject: [PATCH 16/20] GET and HEAD must have body equal to None --- aiohttp/client_reqrep.py | 4 +++- tests/test_pyodide.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index a6617e6e5c4..6bf04a831b7 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -699,8 +699,10 @@ def _make_js_request( from js import Headers, Request # type:ignore[import-not-found] # noqa: I900 from pyodide.ffi import to_js # type:ignore[import-not-found] # noqa: I900 + if method.lower() in ["get", "head"]: + body = None # TODO: to_js does an unnecessary copy. - if isinstance(body, payload.Payload): + elif isinstance(body, payload.Payload): body = to_js(body._value) elif isinstance(body, (bytes, bytearray)): body = to_js(body) diff --git a/tests/test_pyodide.py b/tests/test_pyodide.py index 8d7e62db107..ce2bbc3b56c 100644 --- a/tests/test_pyodide.py +++ b/tests/test_pyodide.py @@ -89,7 +89,7 @@ def fetch_handler(request: JsRequest) -> Future[JsResponse]: "User-Agent": "Python/3.11 aiohttp/4.0.0a2.dev0", } assert request.kwargs["signal"] == "AbortSignal" - assert request.kwargs["body"] == b"" + assert request.kwargs["body"] is None fut = Future() resp = JsResponse( 200, "OK", [["Content-type", "text/html; charset=utf-8"]], JsBuffer(b"abc") From 584d90f19f28e6153e84258b3809a5429de1a7f7 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 5 Dec 2023 13:50:31 -0800 Subject: [PATCH 17/20] Add simple in-pyodide test --- .github/workflows/ci-cd.yml | 16 +++++------- tests/test_pyodide.py | 50 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b91362b1908..81b69a7717b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -333,21 +333,17 @@ jobs: run: | CFLAGS=-g2 LDFLAGS=-g2 pyodide build - - name: set up node - uses: actions/setup-node@v3.8.1 + - uses: pyodide/pyodide-actions/download-pyodide@v1 with: - node-version: ${{ env.NODE_VERSION }} + version: ${{ inputs.pyodide-version }} + to: pyodide-dist - - name: Set up Pyodide virtual environment - run: | - pyodide venv .venv-pyodide - source .venv-pyodide/bin/activate - pip install dist/*.whl + - uses: pyodide/pyodide-actions/install-browser@v1 - name: Test run: | - source .venv-pyodide/bin/activate - + pip install pytest-pyodide + pytest tests/test_pyodide.py --rt chrome --dist-dir ./pyodide-dist check: # This job does nothing and is only used for the branch protection if: always() diff --git a/tests/test_pyodide.py b/tests/test_pyodide.py index ce2bbc3b56c..9412d47c082 100644 --- a/tests/test_pyodide.py +++ b/tests/test_pyodide.py @@ -1,11 +1,19 @@ +import shutil import sys from asyncio import Future from collections.abc import Mapping +from pathlib import Path from types import ModuleType from typing import Any +import pytest from pytest import fixture +try: + from pytest_pyodide import run_in_pyodide # noqa: I900 +except ImportError: + run_in_pyodide = pytest.mark.skip("pytest-pyodide not installed") + from aiohttp import ClientSession, client, client_reqrep, connector from aiohttp.connector import PyodideConnector @@ -104,3 +112,45 @@ def fetch_handler(request: JsRequest) -> Future[JsResponse]: assert response.headers["content-type"] == "text/html; charset=utf-8" html = await response.text() assert html == "abc" + + +@fixture +def install_aiohttp(selenium, request): + wheel = next(Path("dist").glob("*.whl")) + dist_dir = request.config.option.dist_dir + dist_wheel = dist_dir / wheel.name + shutil.copyfile(wheel, dist_wheel) + selenium.load_package(["multidict", "yarl", "aiosignal"]) + selenium.load_package(wheel.name) + try: + yield + finally: + dist_wheel.unlink() + + +@fixture +async def url_to_fetch(request, web_server_main): + target_file = Path(request.config.option.dist_dir) / "test.txt" + target_file.write_text("hello there!") + server_host, server_port, _ = web_server_main + try: + yield f"http://{server_host}:{server_port}/{target_file.name}" + finally: + target_file.unlink() + + +@fixture +async def loop_wrapper(loop): + return None + + +@run_in_pyodide +async def test_pyodide(selenium, install_aiohttp, url_to_fetch, loop_wrapper) -> None: + from aiohttp import ClientSession + + async with ClientSession() as session: + async with session.get(url_to_fetch) as response: + assert response.status == 200 + assert response.headers["content-type"] == "text/plain" + html = await response.text() + assert html == "hello there!" From 7017b3fb68e11cb7d53da2b5cbcdb0d917ebb53e Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 5 Dec 2023 13:58:40 -0800 Subject: [PATCH 18/20] Add self to CONTRIBUTORS --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 539a8807689..d95c3493b6d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -141,6 +141,7 @@ Hans Adema Harmon Y. Harry Liu Hiroshi Ogawa +Hood Chatham Hrishikesh Paranjape Hu Bo Hugh Young From 819ef145aa97ef73955880bfca24353b873ddca5 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 5 Dec 2023 14:00:13 -0800 Subject: [PATCH 19/20] Add CHANGES file --- CHANGES/7803.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/7803.feature diff --git a/CHANGES/7803.feature b/CHANGES/7803.feature new file mode 100644 index 00000000000..68e0e864b9d --- /dev/null +++ b/CHANGES/7803.feature @@ -0,0 +1 @@ +Added basic support for using aiohttp in Pyodide. From d9577b8c408f7d94196bb1c5627eb3b62414c5e6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 8 Dec 2023 17:32:02 -0800 Subject: [PATCH 20/20] Fix download-pyodide invocation --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 81b69a7717b..67e4c2adc7b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -335,7 +335,7 @@ jobs: - uses: pyodide/pyodide-actions/download-pyodide@v1 with: - version: ${{ inputs.pyodide-version }} + version: ${{ env.PYODIDE_VERSION }} to: pyodide-dist - uses: pyodide/pyodide-actions/install-browser@v1