From 16dc0909cee7024d8d6b03b7a571597e74cdd2f6 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 12:52:13 -0300 Subject: [PATCH 1/7] requests: Add _http module for response parsing. Introduce read_headers(), read_body(), and chunked decoding without changing the public request() API yet. Signed-off-by: Pablo Ventura --- python-ecosys/requests/requests/_http.py | 140 +++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 python-ecosys/requests/requests/_http.py diff --git a/python-ecosys/requests/requests/_http.py b/python-ecosys/requests/requests/_http.py new file mode 100644 index 000000000..57cc007e4 --- /dev/null +++ b/python-ecosys/requests/requests/_http.py @@ -0,0 +1,140 @@ +# HTTP/1.x response parsing helpers for the requests package. + +DEFAULT_MAX_BODY = 32 * 1024 + + +class Headers: + """HTTP headers with case-insensitive keys.""" + + def __init__(self): + self._data = {} + + def __setitem__(self, key, value): + self._data[key.lower()] = value + + def __getitem__(self, key): + return self._data[key.lower()] + + def get(self, key, default=None): + return self._data.get(key.lower(), default) + + +def read_status_line(stream): + line = stream.readline() + if not line: + raise ValueError("HTTP error: empty status line") + parts = line.split(None, 2) + if len(parts) < 2: + raise ValueError("HTTP error: BadStatusLine:\n%s" % parts) + status = int(parts[1]) + reason = parts[2].rstrip() if len(parts) > 2 else "" + return status, reason + + +def read_headers(stream, parse_headers=None): + headers = Headers() + while True: + line = stream.readline() + if not line or line == b"\r\n": + break + if parse_headers is False: + continue + if parse_headers is True: + text = str(line, "utf-8") + key, value = text.split(":", 1) + headers[key] = value.strip() + else: + parse_headers(line, headers) + return headers + + +def _read_exact(stream, length, max_remaining): + out = b"" + while length > 0: + if max_remaining is not None and max_remaining <= 0: + raise ValueError("Response body exceeds max_body limit") + chunk_len = length + if max_remaining is not None and chunk_len > max_remaining: + chunk_len = max_remaining + chunk = stream.read(chunk_len) + if not chunk: + break + out += chunk + length -= len(chunk) + if max_remaining is not None: + max_remaining -= len(chunk) + if length > 0: + raise ValueError("Connection closed before Content-Length satisfied") + return out, max_remaining + + +def _read_chunked(stream, max_remaining): + out = b"" + while True: + line = stream.readline() + if not line: + raise ValueError("Connection closed in chunked body") + size_line = line.strip().split(b";", 1)[0] + size = int(size_line, 16) + if size == 0: + stream.readline() + break + if max_remaining is not None and size > max_remaining: + raise ValueError("Response body exceeds max_body limit") + chunk, max_remaining = _read_exact(stream, size, max_remaining) + out += chunk + stream.readline() + return out, max_remaining + + +def _read_until_close(stream, max_remaining): + out = b"" + while True: + if max_remaining is not None and max_remaining <= 0: + raise ValueError("Response body exceeds max_body limit") + to_read = 256 + if max_remaining is not None: + to_read = min(to_read, max_remaining) + chunk = stream.read(to_read) + if not chunk: + break + out += chunk + if max_remaining is not None: + max_remaining -= len(chunk) + return out, max_remaining + + +def read_body(stream, headers, max_body=None): + """Read the response body according to Content-Length or chunked encoding.""" + max_remaining = max_body + encoding = headers.get("transfer-encoding", "") + if encoding and "chunked" in encoding.lower(): + body, _ = _read_chunked(stream, max_remaining) + return body + content_length = headers.get("content-length") + if content_length is not None: + length = int(content_length) + body, _ = _read_exact(stream, length, max_remaining) + return body + body, _ = _read_until_close(stream, max_remaining) + return body + + +def resolve_redirect_url(base_url, location): + """Resolve a redirect Location header against the request URL.""" + loc = location.strip() + if loc.startswith("http:") or loc.startswith("https:"): + return loc + try: + proto, dummy, host, path = base_url.split("/", 3) + except ValueError: + proto, dummy, host = base_url.split("/", 2) + path = "" + if loc.startswith("/"): + return proto + "//" + host + loc + if path and "/" in path: + dir_path = path.rsplit("/", 1)[0] + new_path = dir_path + "/" + loc + else: + new_path = loc + return proto + "//" + host + "/" + new_path From 14c507618e1faa10f6451483eb45bdbdc28d8b96 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 12:55:11 -0300 Subject: [PATCH 2/7] requests: Read response bodies and use HTTP/1.1. Parse Content-Length and chunked Transfer-Encoding; store body on Response.content. Default request line to HTTP/1.1. Add max_body limit (32 KiB default). Fix relative redirect Location URLs. Signed-off-by: Pablo Ventura --- python-ecosys/requests/requests/__init__.py | 142 +++++++++++--------- 1 file changed, 76 insertions(+), 66 deletions(-) diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 68b4b18cb..9c22d2ee4 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -1,26 +1,32 @@ import socket +from ._http import ( + DEFAULT_MAX_BODY, + read_body, + read_headers, + read_status_line, + resolve_redirect_url, +) + + +def _to_bytes(val): + if isinstance(val, str): + return val.encode() + return val + class Response: - def __init__(self, f): - self.raw = f + def __init__(self, body=b""): + self.raw = None self.encoding = "utf-8" - self._cached = None + self._cached = body def close(self): - if self.raw: - self.raw.close() - self.raw = None + self.raw = None self._cached = None @property def content(self): - if self._cached is None: - try: - self._cached = self.raw.read() - finally: - self.raw.close() - self.raw = None return self._cached @property @@ -43,20 +49,21 @@ def request( auth=None, timeout=None, parse_headers=True, + max_body=DEFAULT_MAX_BODY, ): if headers is None: headers = {} else: headers = headers.copy() - redirect = None # redirection url, None means no redirection + redirect = None chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) if auth is not None: import binascii username, password = auth - formatted = b"{}:{}".format(username, password) + formatted = (username + ":" + password).encode() formatted = str(binascii.b2a_base64(formatted)[:-1], "ascii") headers["Authorization"] = "Basic {}".format(formatted) @@ -81,15 +88,9 @@ def request( ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] - resp_d = None - if parse_headers is not False: - resp_d = {} - s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2]) if timeout is not None: - # Note: settimeout is not supported on all platforms, will raise - # an AttributeError if not available. s.settimeout(timeout) try: @@ -98,7 +99,11 @@ def request( context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) context.verify_mode = tls.CERT_NONE s = context.wrap_socket(s, server_hostname=host) - s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) + if not isinstance(method, bytes): + method = method.encode() + if not isinstance(path, bytes): + path = path.encode() + s.write(b"%s /%s HTTP/1.1\r\n" % (method, path)) if "Host" not in headers: headers["Host"] = host @@ -122,11 +127,10 @@ def request( if "Connection" not in headers: headers["Connection"] = "close" - # Iterate over keys to avoid tuple alloc for k in headers: - s.write(k) + s.write(_to_bytes(k)) s.write(b": ") - s.write(headers[k]) + s.write(_to_bytes(headers[k])) s.write(b"\r\n") s.write(b"\r\n") @@ -135,66 +139,72 @@ def request( if chunked_data: if headers.get("Transfer-Encoding", None) == "chunked": for chunk in data: + chunk = _to_bytes(chunk) s.write(b"%x\r\n" % len(chunk)) s.write(chunk) s.write(b"\r\n") - s.write("0\r\n\r\n") + s.write(b"0\r\n\r\n") else: for chunk in data: - s.write(chunk) + s.write(_to_bytes(chunk)) else: - s.write(data) - - l = s.readline() - # print(l) - l = l.split(None, 2) - if len(l) < 2: - # Invalid response - raise ValueError("HTTP error: BadStatusLine:\n%s" % l) - status = int(l[1]) - reason = "" - if len(l) > 2: - reason = l[2].rstrip() - while True: - l = s.readline() - if not l or l == b"\r\n": - break - # print(l) - if l.startswith(b"Transfer-Encoding:"): - if b"chunked" in l: - raise ValueError("Unsupported " + str(l, "utf-8")) - elif l.startswith(b"Location:") and not 200 <= status <= 299: + s.write(_to_bytes(data)) + + status, reason = read_status_line(s) + resp_headers = read_headers(s, parse_headers) + + if not 200 <= status <= 299: + location = resp_headers.get("location") + if location: if status in [301, 302, 303, 307, 308]: - redirect = str(l[10:-2], "utf-8") + redirect = resolve_redirect_url(url, location) else: raise NotImplementedError("Redirect %d not yet supported" % status) - if parse_headers is False: - pass - elif parse_headers is True: - l = str(l, "utf-8") - k, v = l.split(":", 1) - resp_d[k] = v.strip() - else: - parse_headers(l, resp_d) except OSError: s.close() raise if redirect: s.close() - # Use the host specified in the redirect URL, as it may not be the same as the original URL. headers.pop("Host", None) if status in [301, 302, 303]: - return request("GET", redirect, None, None, headers, stream) - else: - return request(method, redirect, data, json, headers, stream) + return request( + "GET", + redirect, + None, + None, + headers, + stream, + auth, + timeout, + parse_headers, + max_body, + ) + return request( + method, + redirect, + data, + json, + headers, + stream, + auth, + timeout, + parse_headers, + max_body, + ) + + if method == "HEAD": + body = b"" else: - resp = Response(s) - resp.status_code = status - resp.reason = reason - if resp_d is not None: - resp.headers = resp_d - return resp + body = read_body(s, resp_headers, max_body) + resp = Response(body) + resp.raw = s + s.close() + resp.status_code = status + resp.reason = reason + if parse_headers is not False: + resp.headers = resp_headers + return resp def head(url, **kw): From e17ee734a79f67c6ffc84ef01f2cd42176b73355 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 12:55:16 -0300 Subject: [PATCH 3/7] requests: Add tests for response bodies and HTTP/1.1. Cover Content-Length, chunked encoding, max_body, and relative redirects using the socket mock. Signed-off-by: Pablo Ventura --- python-ecosys/requests/test_requests.py | 102 +++++++++++++++++++----- 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index ac77291b0..0d10e4249 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -3,9 +3,9 @@ class Socket: - def __init__(self): + def __init__(self, read_data=b"HTTP/1.1 200 OK\r\n\r\n"): self._write_buffer = io.BytesIO() - self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n") + self._read_buffer = io.BytesIO(read_data) def connect(self, address): pass @@ -16,6 +16,12 @@ def write(self, buf): def readline(self): return self._read_buffer.readline() + def read(self, size=-1): + return self._read_buffer.read(size) + + def close(self): + pass + class socket: AF_INET = 2 @@ -43,8 +49,9 @@ def test_simple_get(): response = requests.request("GET", "http://example.com") assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n" + b"GET / HTTP/1.1\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" ), format_message(response) + assert response.content == b"" def test_get_auth(): @@ -53,9 +60,9 @@ def test_get_auth(): ) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Host: example.com\r\n" + b"GET / HTTP/1.1\r\n" + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" + + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" ), format_message(response) @@ -64,7 +71,7 @@ def test_get_custom_header(): response = requests.request("GET", "http://example.com", headers={"User-Agent": "test-agent"}) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"GET / HTTP/1.1\r\n" + b"User-Agent: test-agent\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" @@ -75,11 +82,11 @@ def test_post_json(): response = requests.request("GET", "http://example.com", json="test") assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Connection: close\r\n" - + b"Content-Type: application/json\r\n" + b"GET / HTTP/1.1\r\n" + b"Host: example.com\r\n" - + b"Content-Length: 6\r\n\r\n" + + b"Content-Type: application/json\r\n" + + b"Content-Length: 6\r\n" + + b"Connection: close\r\n\r\n" + b'"test"' ), format_message(response) @@ -91,9 +98,9 @@ def chunks(): response = requests.request("GET", "http://example.com", data=chunks()) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Transfer-Encoding: chunked\r\n" + b"GET / HTTP/1.1\r\n" + b"Host: example.com\r\n" + + b"Transfer-Encoding: chunked\r\n" + b"Connection: close\r\n\r\n" + b"4\r\ntest\r\n" + b"0\r\n\r\n" @@ -106,7 +113,7 @@ def test_overwrite_get_headers(): ) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n" + b"GET / HTTP/1.1\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n" ), format_message(response) @@ -119,11 +126,11 @@ def test_overwrite_post_json_headers(): ) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Connection: close\r\n" - + b"Content-Length: 10\r\n" + b"GET / HTTP/1.1\r\n" + b"Content-Type: text/plain\r\n" - + b"Host: example.com\r\n\r\n" + + b"Content-Length: 10\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + b'"test"' ), format_message(response) @@ -137,9 +144,9 @@ def chunks(): ) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Host: example.com\r\n" + b"GET / HTTP/1.1\r\n" + b"Content-Length: 4\r\n" + + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" + b"test" ), format_message(response) @@ -153,6 +160,58 @@ def test_do_not_modify_headers_argument(): assert do_not_modify_this_dict == {}, do_not_modify_this_dict +def test_content_length_body(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello" + ) + response = requests.request("GET", "http://example.com") + assert response.content == b"hello" + assert response.headers["content-length"] == "5" + + +def test_chunked_body(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + ) + response = requests.request("GET", "http://example.com") + assert response.content == b"hello" + + +def test_case_insensitive_headers(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK" + ) + response = requests.request("GET", "http://example.com") + assert response.headers["content-length"] == "2" + assert response.headers["Content-Length"] == "2" + + +def test_max_body_limit(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n" + b"x" * 10 + ) + try: + requests.request("GET", "http://example.com", max_body=5) + assert False, "expected ValueError" + except ValueError as e: + assert "max_body" in str(e) + + +def test_relative_redirect(): + calls = [] + + def socket_factory(*a, **k): + if not calls: + calls.append(1) + return Socket(read_data=b"HTTP/1.1 302 Found\r\nLocation: /other\r\n\r\n") + return Socket(read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + + socket.socket = socket_factory + response = requests.request("GET", "http://example.com/path/here") + assert response.content == b"ok" + assert b"GET /other HTTP/1.1" in response.raw._write_buffer.getvalue() + + test_simple_get() test_get_auth() test_get_custom_header() @@ -162,3 +221,8 @@ def test_do_not_modify_headers_argument(): test_overwrite_post_json_headers() test_overwrite_post_chunked_data_headers() test_do_not_modify_headers_argument() +test_content_length_body() +test_chunked_body() +test_case_insensitive_headers() +test_max_body_limit() +test_relative_redirect() From 60bf1bcf874eef5ae7f728e1e4dad0ed0ad9ef96 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 12:55:16 -0300 Subject: [PATCH 4/7] requests: Update README and bump package version to 0.11.0. Document HTTP/1.1 response support, max_body, and remaining limits. Signed-off-by: Pablo Ventura --- python-ecosys/requests/README.md | 14 ++++++++++++-- python-ecosys/requests/manifest.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md index d6ceaadc5..da27cbc16 100644 --- a/python-ecosys/requests/README.md +++ b/python-ecosys/requests/README.md @@ -4,7 +4,9 @@ This module provides a lightweight version of the Python [requests](https://requests.readthedocs.io/en/latest/) library. It includes support for all HTTP verbs, https, json decoding of responses, -redirects, basic authentication. +redirects (including relative Location URLs), basic authentication, HTTP/1.1 +requests, and reading response bodies with Content-Length or chunked +Transfer-Encoding. ### Limitations @@ -13,4 +15,12 @@ redirects, basic authentication. multipart-form encoding of post data (this can be done manually). * Compressed requests/responses are not currently supported. * File upload is not supported. -* Chunked encoding in responses is not supported. +* HTTP keep-alive connection reuse is not supported (Connection: close by default). +* Response bodies are buffered in memory. By default at most 32 KiB are read + per response; pass ``max_body=None`` to disable the limit or set another + byte count with ``max_body=N``. + +### Follow-up work + +* TLS certificate verification (see micropython-lib issue #838). +* ``stream=True`` incremental body reads (see issue #777). diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 85f159753..1571a29d9 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.2", pypi="requests") +metadata(version="0.11.0", pypi="requests") package("requests") From e6beaa903b3462189c16a66e1f77f32379201314 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 13:05:25 -0300 Subject: [PATCH 5/7] requests: Expand tests and fix MicroPython compatibility. Order-independent mock asserts, live TCP tests, unix/ESP32 runners, prefer /lib over frozen requests on ESP32, and HEAD/auth/encoding fixes. Signed-off-by: Pablo Ventura --- python-ecosys/requests/requests/__init__.py | 4 +- python-ecosys/requests/requests/_http.py | 8 +- python-ecosys/requests/run-esp32-tests.sh | 26 +++ python-ecosys/requests/run-unix-tests.sh | 21 ++ python-ecosys/requests/test_requests.py | 229 ++++++++++++------- python-ecosys/requests/test_requests_live.py | 150 ++++++++++++ 6 files changed, 349 insertions(+), 89 deletions(-) create mode 100755 python-ecosys/requests/run-esp32-tests.sh create mode 100755 python-ecosys/requests/run-unix-tests.sh create mode 100644 python-ecosys/requests/test_requests_live.py diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 9c22d2ee4..514ae9967 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -63,7 +63,7 @@ def request( import binascii username, password = auth - formatted = (username + ":" + password).encode() + formatted = _to_bytes(username) + b":" + _to_bytes(password) formatted = str(binascii.b2a_base64(formatted)[:-1], "ascii") headers["Authorization"] = "Basic {}".format(formatted) @@ -193,7 +193,7 @@ def request( max_body, ) - if method == "HEAD": + if method == b"HEAD": body = b"" else: body = read_body(s, resp_headers, max_body) diff --git a/python-ecosys/requests/requests/_http.py b/python-ecosys/requests/requests/_http.py index 57cc007e4..8e618ad84 100644 --- a/python-ecosys/requests/requests/_http.py +++ b/python-ecosys/requests/requests/_http.py @@ -108,9 +108,11 @@ def read_body(stream, headers, max_body=None): """Read the response body according to Content-Length or chunked encoding.""" max_remaining = max_body encoding = headers.get("transfer-encoding", "") - if encoding and "chunked" in encoding.lower(): - body, _ = _read_chunked(stream, max_remaining) - return body + if encoding: + enc = encoding if isinstance(encoding, str) else str(encoding, "utf-8") + if "chunked" in enc.lower(): + body, _ = _read_chunked(stream, max_remaining) + return body content_length = headers.get("content-length") if content_length is not None: length = int(content_length) diff --git a/python-ecosys/requests/run-esp32-tests.sh b/python-ecosys/requests/run-esp32-tests.sh new file mode 100755 index 000000000..5409430f6 --- /dev/null +++ b/python-ecosys/requests/run-esp32-tests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Deploy requests package to ESP32 and run mock + live tests via mpremote. +set -euo pipefail + +PORT="${PORT:-/dev/ttyACM0}" +MPREMOTE="${MPREMOTE:-pipx run mpremote}" +ROOT="$(cd "$(dirname "$0")" && pwd)" + +run_mp() { + $MPREMOTE connect "$PORT" "$@" +} + +cd "$ROOT" +echo "== Deploy requests to :lib/requests ==" +run_mp fs mkdir :lib 2>/dev/null || true +run_mp fs mkdir :lib/requests 2>/dev/null || true +run_mp fs cp requests/__init__.py :lib/requests/__init__.py +run_mp fs cp requests/_http.py :lib/requests/_http.py + +echo "== Mock tests ==" +run_mp run test_requests.py + +echo "== Live tests ==" +run_mp run test_requests_live.py + +echo "== ESP32 tests passed ==" diff --git a/python-ecosys/requests/run-unix-tests.sh b/python-ecosys/requests/run-unix-tests.sh new file mode 100755 index 000000000..1d02554a7 --- /dev/null +++ b/python-ecosys/requests/run-unix-tests.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run requests package tests on MicroPython unix port (mock + live). +set -euo pipefail + +MICROPYTHON="${MICROPYTHON:-../../../micropython/ports/unix/build-standard/micropython}" +ROOT="$(cd "$(dirname "$0")" && pwd)" + +if [[ ! -x "$MICROPYTHON" ]]; then + echo "Build unix port first:" >&2 + echo " cd micropython/ports/unix && make submodules && make" >&2 + exit 1 +fi + +cd "$ROOT" +echo "== Mock tests (test_requests.py) ==" +"$MICROPYTHON" test_requests.py + +echo "== Live tests (test_requests_live.py) ==" +"$MICROPYTHON" test_requests_live.py + +echo "== All unix tests passed ==" diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index 0d10e4249..15092896d 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -2,6 +2,24 @@ import sys +def _prefer_filesystem_requests(): + """Use /lib/requests on device instead of older frozen copy (ESP32).""" + try: + import os + + os.stat("/lib/requests/__init__.py") + if "/lib" not in sys.path[:1]: + sys.path.insert(0, "/lib") + except OSError: + pass + + +_prefer_filesystem_requests() + +# Save real socket module before tests replace sys.modules["socket"]. +_real_socket_mod = sys.modules.get("socket") + + class Socket: def __init__(self, read_data=b"HTTP/1.1 200 OK\r\n\r\n"): self._write_buffer = io.BytesIO() @@ -27,6 +45,7 @@ class socket: AF_INET = 2 SOCK_STREAM = 1 IPPROTO_TCP = 6 + SO_REUSEADDR = 4 @staticmethod def getaddrinfo(host, port, af=0, type=0, flags=0): @@ -36,128 +55,149 @@ def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): return Socket() -sys.modules["socket"] = socket +def install_mock_socket(): + sys.modules["socket"] = socket + + +def restore_socket(): + if _real_socket_mod is not None: + sys.modules["socket"] = _real_socket_mod + + +install_mock_socket() # ruff: noqa: E402 import requests -def format_message(response): - return response.raw._write_buffer.getvalue().decode("utf8") +def request_bytes(response): + return response.raw._write_buffer.getvalue() + + +def assert_has(data, *parts): + for part in parts: + if part not in data: + raise AssertionError("missing {!r} in:\n{}".format(part, data)) def test_simple_get(): + install_mock_socket() response = requests.request("GET", "http://example.com") - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + b"\r\n", + ) assert response.content == b"" def test_get_auth(): + install_mock_socket() response = requests.request( "GET", "http://example.com", auth=("test-username", "test-password") ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + ) def test_get_custom_header(): + install_mock_socket() response = requests.request("GET", "http://example.com", headers={"User-Agent": "test-agent"}) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"User-Agent: test-agent\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"User-Agent: test-agent\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + ) def test_post_json(): + install_mock_socket() response = requests.request("GET", "http://example.com", json="test") - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"Host: example.com\r\n" - + b"Content-Type: application/json\r\n" - + b"Content-Length: 6\r\n" - + b"Connection: close\r\n\r\n" - + b'"test"' - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Content-Type: application/json\r\n", + b"Host: example.com\r\n", + b"Content-Length: 6\r\n", + b"Connection: close\r\n", + b'"test"', + ) def test_post_chunked_data(): def chunks(): yield "test" + install_mock_socket() response = requests.request("GET", "http://example.com", data=chunks()) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"Host: example.com\r\n" - + b"Transfer-Encoding: chunked\r\n" - + b"Connection: close\r\n\r\n" - + b"4\r\ntest\r\n" - + b"0\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Transfer-Encoding: chunked\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + b"4\r\ntest\r\n", + b"0\r\n\r\n", + ) def test_overwrite_get_headers(): + install_mock_socket() response = requests.request( "GET", "http://example.com", headers={"Host": "test.com", "Connection": "keep-alive"} ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has(data, b"GET / HTTP/1.1\r\n", b"Host: test.com\r\n", b"Connection: keep-alive\r\n") def test_overwrite_post_json_headers(): + install_mock_socket() response = requests.request( "GET", "http://example.com", json="test", headers={"Content-Type": "text/plain", "Content-Length": "10"}, ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"Content-Type: text/plain\r\n" - + b"Content-Length: 10\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - + b'"test"' - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"Content-Type: text/plain\r\n", + b"Content-Length: 10\r\n", + b"Host: example.com\r\n", + b'"test"', + ) def test_overwrite_post_chunked_data_headers(): def chunks(): yield "test" + install_mock_socket() response = requests.request( "GET", "http://example.com", data=chunks(), headers={"Content-Length": "4"} ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.1\r\n" - + b"Content-Length: 4\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - + b"test" - ), format_message(response) + data = request_bytes(response) + assert_has(data, b"Content-Length: 4\r\n", b"Host: example.com\r\n", b"test") def test_do_not_modify_headers_argument(): - global do_not_modify_this_dict - do_not_modify_this_dict = {} - requests.request("GET", "http://example.com", headers=do_not_modify_this_dict) - - assert do_not_modify_this_dict == {}, do_not_modify_this_dict + install_mock_socket() + headers_arg = {} + requests.request("GET", "http://example.com", headers=headers_arg) + assert headers_arg == {}, headers_arg def test_content_length_body(): @@ -167,6 +207,7 @@ def test_content_length_body(): response = requests.request("GET", "http://example.com") assert response.content == b"hello" assert response.headers["content-length"] == "5" + install_mock_socket() def test_chunked_body(): @@ -175,6 +216,7 @@ def test_chunked_body(): ) response = requests.request("GET", "http://example.com") assert response.content == b"hello" + install_mock_socket() def test_case_insensitive_headers(): @@ -184,17 +226,23 @@ def test_case_insensitive_headers(): response = requests.request("GET", "http://example.com") assert response.headers["content-length"] == "2" assert response.headers["Content-Length"] == "2" + install_mock_socket() def test_max_body_limit(): socket.socket = lambda *a, **k: Socket( read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n" + b"x" * 10 ) + raised = False try: requests.request("GET", "http://example.com", max_body=5) - assert False, "expected ValueError" except ValueError as e: - assert "max_body" in str(e) + raised = True + if "max_body" not in str(e): + raise + if not raised: + raise AssertionError("expected ValueError for max_body") + install_mock_socket() def test_relative_redirect(): @@ -209,20 +257,33 @@ def socket_factory(*a, **k): socket.socket = socket_factory response = requests.request("GET", "http://example.com/path/here") assert response.content == b"ok" - assert b"GET /other HTTP/1.1" in response.raw._write_buffer.getvalue() - - -test_simple_get() -test_get_auth() -test_get_custom_header() -test_post_json() -test_post_chunked_data() -test_overwrite_get_headers() -test_overwrite_post_json_headers() -test_overwrite_post_chunked_data_headers() -test_do_not_modify_headers_argument() -test_content_length_body() -test_chunked_body() -test_case_insensitive_headers() -test_max_body_limit() -test_relative_redirect() + assert b"GET /other HTTP/1.1" in request_bytes(response) + install_mock_socket() + + +TESTS = ( + test_simple_get, + test_get_auth, + test_get_custom_header, + test_post_json, + test_post_chunked_data, + test_overwrite_get_headers, + test_overwrite_post_json_headers, + test_overwrite_post_chunked_data_headers, + test_do_not_modify_headers_argument, + test_content_length_body, + test_chunked_body, + test_case_insensitive_headers, + test_max_body_limit, + test_relative_redirect, +) + + +def run_all(): + for test in TESTS: + test() + + +if __name__ == "__main__": + run_all() + print("test_requests: {} tests OK".format(len(TESTS))) diff --git a/python-ecosys/requests/test_requests_live.py b/python-ecosys/requests/test_requests_live.py new file mode 100644 index 000000000..a3482d58d --- /dev/null +++ b/python-ecosys/requests/test_requests_live.py @@ -0,0 +1,150 @@ +"""Live HTTP tests using a local TCP server (unix port / network stack). + +Run with MicroPython unix: + micropython test_requests_live.py +""" + +import sys +import time + +try: + import _thread +except ImportError: + _thread = None + + +def _prefer_filesystem_requests(): + try: + import os + + os.stat("/lib/requests/__init__.py") + if "/lib" not in sys.path[:1]: + sys.path.insert(0, "/lib") + except OSError: + pass + + +_prefer_filesystem_requests() + +if "requests" in sys.modules: + del sys.modules["requests"] + +import socket + +import requests + +_next_port = 52000 + + +def _alloc_port(): + global _next_port + port = _next_port + _next_port += 1 + return port + + +def _bind_addr(port): + return socket.getaddrinfo("127.0.0.1", port)[0][-1] + + +def _drain_request(conn): + while True: + line = conn.readline() + if not line or line == b"\r\n": + break + + +def _run_server(port, response_bytes): + srv = socket.socket() + srv.bind(_bind_addr(port)) + srv.listen(1) + srv.settimeout(10) + conn, addr = srv.accept() + _drain_request(conn) + conn.send(response_bytes) + conn.close() + srv.close() + + +def _fetch(port, path="/", method="GET", **kw): + url = "http://127.0.0.1:{}{}".format(port, path) + return requests.request(method, url, **kw) + + +def test_live_content_length(): + port = _alloc_port() + body = b"live-ok" + resp_bytes = b"HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\n" + body + _thread.start_new_thread(_run_server, (port, resp_bytes)) + time.sleep(0.2) + r = _fetch(port) + assert r.status_code == 200 + assert r.content == body + + +def test_live_chunked(): + port = _alloc_port() + resp_bytes = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + _thread.start_new_thread(_run_server, (port, resp_bytes)) + time.sleep(0.2) + r = _fetch(port) + assert r.content == b"hello" + + +def test_live_redirect_relative(): + port = _alloc_port() + first = b"HTTP/1.1 302 Found\r\nLocation: /done\r\nConnection: close\r\n\r\n" + second = b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nyes" + + def multi_server(): + srv = socket.socket() + srv.bind(_bind_addr(port)) + srv.listen(1) + srv.settimeout(10) + for resp in (first, second): + conn, addr = srv.accept() + _drain_request(conn) + conn.send(resp) + conn.close() + srv.close() + + _thread.start_new_thread(multi_server, ()) + time.sleep(0.2) + r = _fetch(port, "/start") + assert r.content == b"yes" + + +def test_live_max_body(): + port = _alloc_port() + body = b"0123456789" + resp_bytes = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n" + body + _thread.start_new_thread(_run_server, (port, resp_bytes)) + time.sleep(0.2) + raised = False + try: + _fetch(port, max_body=5) + except ValueError: + raised = True + if not raised: + raise AssertionError("expected max_body ValueError") + + +LIVE_TESTS = ( + test_live_content_length, + test_live_chunked, + test_live_redirect_relative, + test_live_max_body, +) + + +def run_all(): + if _thread is None: + print("test_requests_live: SKIP (no _thread)") + return + for test in LIVE_TESTS: + test() + + +if __name__ == "__main__": + run_all() + print("test_requests_live: {} tests OK".format(len(LIVE_TESTS))) From 4b9403706a8065a4643eba2eabbfead3cc325071 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 13:05:45 -0300 Subject: [PATCH 6/7] requests: Fix unix test runner path resolution. Signed-off-by: Pablo Ventura --- python-ecosys/requests/run-unix-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-ecosys/requests/run-unix-tests.sh b/python-ecosys/requests/run-unix-tests.sh index 1d02554a7..cdf02c06c 100755 --- a/python-ecosys/requests/run-unix-tests.sh +++ b/python-ecosys/requests/run-unix-tests.sh @@ -2,8 +2,8 @@ # Run requests package tests on MicroPython unix port (mock + live). set -euo pipefail -MICROPYTHON="${MICROPYTHON:-../../../micropython/ports/unix/build-standard/micropython}" ROOT="$(cd "$(dirname "$0")" && pwd)" +MICROPYTHON="${MICROPYTHON:-$ROOT/../../../micropython/ports/unix/build-standard/micropython}" if [[ ! -x "$MICROPYTHON" ]]; then echo "Build unix port first:" >&2 From be5ea1e7934465f14ac18bc00a6854cd188df033 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 1 Jun 2026 13:13:48 -0300 Subject: [PATCH 7/7] requests: Silence ruff E402 in live test imports. Signed-off-by: Pablo Ventura --- python-ecosys/requests/test_requests_live.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python-ecosys/requests/test_requests_live.py b/python-ecosys/requests/test_requests_live.py index a3482d58d..60d8309ec 100644 --- a/python-ecosys/requests/test_requests_live.py +++ b/python-ecosys/requests/test_requests_live.py @@ -29,6 +29,7 @@ def _prefer_filesystem_requests(): if "requests" in sys.modules: del sys.modules["requests"] +# ruff: noqa: E402 import socket import requests