requests: HTTP/1.1 response bodies and max_body limit

python-ecosys/requests/README.md

@@ -4,7 +4,9 @@ This module provides a lightweight version of the Python
 [requests](https://requests.readthedocs.io/en/latest/) library.
 
 It includes support for all HTTP verbs, https, json decoding of responses,
-redirects, basic authentication.
+redirects (including relative Location URLs), basic authentication, HTTP/1.1
+requests, and reading response bodies with Content-Length or chunked
+Transfer-Encoding.
 
 ### Limitations
 
@@ -13,4 +15,12 @@ redirects, basic authentication.
   multipart-form encoding of post data (this can be done manually).
 * Compressed requests/responses are not currently supported.
 * File upload is not supported.
-* Chunked encoding in responses is not supported.
+* HTTP keep-alive connection reuse is not supported (Connection: close by default).
+* Response bodies are buffered in memory. By default at most 32 KiB are read
+  per response; pass ``max_body=None`` to disable the limit or set another
+  byte count with ``max_body=N``.
+
+### Follow-up work
+
+* TLS certificate verification (see micropython-lib issue #838).
+* ``stream=True`` incremental body reads (see issue #777).

python-ecosys/requests/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.10.2", pypi="requests")
+metadata(version="0.11.0", pypi="requests")
 
 package("requests")

python-ecosys/requests/requests/__init__.py

@@ -1,26 +1,32 @@
 import socket
 
+from ._http import (
+    DEFAULT_MAX_BODY,
+    read_body,
+    read_headers,
+    read_status_line,
+    resolve_redirect_url,
+)
+
+
+def _to_bytes(val):
+    if isinstance(val, str):
+        return val.encode()
+    return val
+
 
 class Response:
-    def __init__(self, f):
-        self.raw = f
+    def __init__(self, body=b""):
+        self.raw = None
         self.encoding = "utf-8"
-        self._cached = None
+        self._cached = body
 
     def close(self):
-        if self.raw:
-            self.raw.close()
-            self.raw = None
+        self.raw = None
         self._cached = None
 
     @property
     def content(self):
-        if self._cached is None:
-            try:
-                self._cached = self.raw.read()
-            finally:
-                self.raw.close()
-                self.raw = None
         return self._cached
 
     @property
@@ -43,20 +49,21 @@ def request(
     auth=None,
     timeout=None,
     parse_headers=True,
+    max_body=DEFAULT_MAX_BODY,
 ):
     if headers is None:
         headers = {}
     else:
         headers = headers.copy()
 
-    redirect = None  # redirection url, None means no redirection
+    redirect = None
     chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None)
 
     if auth is not None:
         import binascii
 
         username, password = auth
-        formatted = b"{}:{}".format(username, password)
+        formatted = _to_bytes(username) + b":" + _to_bytes(password)
         formatted = str(binascii.b2a_base64(formatted)[:-1], "ascii")
         headers["Authorization"] = "Basic {}".format(formatted)
 
@@ -81,15 +88,9 @@ def request(
     ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
     ai = ai[0]
 
-    resp_d = None
-    if parse_headers is not False:
-        resp_d = {}
-
     s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2])
 
     if timeout is not None:
-        # Note: settimeout is not supported on all platforms, will raise
-        # an AttributeError if not available.
         s.settimeout(timeout)
 
     try:
@@ -98,7 +99,11 @@ def request(
             context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
             context.verify_mode = tls.CERT_NONE
             s = context.wrap_socket(s, server_hostname=host)
-        s.write(b"%s /%s HTTP/1.0\r\n" % (method, path))
+        if not isinstance(method, bytes):
+            method = method.encode()
+        if not isinstance(path, bytes):
+            path = path.encode()
+        s.write(b"%s /%s HTTP/1.1\r\n" % (method, path))
 
         if "Host" not in headers:
             headers["Host"] = host
@@ -122,11 +127,10 @@ def request(
         if "Connection" not in headers:
             headers["Connection"] = "close"
 
-        # Iterate over keys to avoid tuple alloc
         for k in headers:
-            s.write(k)
+            s.write(_to_bytes(k))
             s.write(b": ")
-            s.write(headers[k])
+            s.write(_to_bytes(headers[k]))
             s.write(b"\r\n")
 
         s.write(b"\r\n")
@@ -135,66 +139,72 @@ def request(
             if chunked_data:
                 if headers.get("Transfer-Encoding", None) == "chunked":
                     for chunk in data:
+                        chunk = _to_bytes(chunk)
                         s.write(b"%x\r\n" % len(chunk))
                         s.write(chunk)
                         s.write(b"\r\n")
-                    s.write("0\r\n\r\n")
+                    s.write(b"0\r\n\r\n")
                 else:
                     for chunk in data:
-                        s.write(chunk)
+                        s.write(_to_bytes(chunk))
             else:
-                s.write(data)
-
-        l = s.readline()
-        # print(l)
-        l = l.split(None, 2)
-        if len(l) < 2:
-            # Invalid response
-            raise ValueError("HTTP error: BadStatusLine:\n%s" % l)
-        status = int(l[1])
-        reason = ""
-        if len(l) > 2:
-            reason = l[2].rstrip()
-        while True:
-            l = s.readline()
-            if not l or l == b"\r\n":
-                break
-            # print(l)
-            if l.startswith(b"Transfer-Encoding:"):
-                if b"chunked" in l:
-                    raise ValueError("Unsupported " + str(l, "utf-8"))
-            elif l.startswith(b"Location:") and not 200 <= status <= 299:
+                s.write(_to_bytes(data))
+
+        status, reason = read_status_line(s)
+        resp_headers = read_headers(s, parse_headers)
+
+        if not 200 <= status <= 299:
+            location = resp_headers.get("location")
+            if location:
                 if status in [301, 302, 303, 307, 308]:
-                    redirect = str(l[10:-2], "utf-8")
+                    redirect = resolve_redirect_url(url, location)
                 else:
                     raise NotImplementedError("Redirect %d not yet supported" % status)
-            if parse_headers is False:
-                pass
-            elif parse_headers is True:
-                l = str(l, "utf-8")
-                k, v = l.split(":", 1)
-                resp_d[k] = v.strip()
-            else:
-                parse_headers(l, resp_d)
     except OSError:
         s.close()
         raise
 
     if redirect:
         s.close()
-        # Use the host specified in the redirect URL, as it may not be the same as the original URL.
         headers.pop("Host", None)
         if status in [301, 302, 303]:
-            return request("GET", redirect, None, None, headers, stream)
-        else:
-            return request(method, redirect, data, json, headers, stream)
+            return request(
+                "GET",
+                redirect,
+                None,
+                None,
+                headers,
+                stream,
+                auth,
+                timeout,
+                parse_headers,
+                max_body,
+            )
+        return request(
+            method,
+            redirect,
+            data,
+            json,
+            headers,
+            stream,
+            auth,
+            timeout,
+            parse_headers,
+            max_body,
+        )
+
+    if method == b"HEAD":
+        body = b""
     else:
-        resp = Response(s)
-        resp.status_code = status
-        resp.reason = reason
-        if resp_d is not None:
-            resp.headers = resp_d
-        return resp
+        body = read_body(s, resp_headers, max_body)
+    resp = Response(body)
+    resp.raw = s
+    s.close()
+    resp.status_code = status
+    resp.reason = reason
+    if parse_headers is not False:
+        resp.headers = resp_headers
+    return resp
 
 
 def head(url, **kw):

python-ecosys/requests/requests/_http.py

@@ -0,0 +1,142 @@
+# HTTP/1.x response parsing helpers for the requests package.
+
+DEFAULT_MAX_BODY = 32 * 1024
+
+
+class Headers:
+    """HTTP headers with case-insensitive keys."""
+
+    def __init__(self):
+        self._data = {}
+
+    def __setitem__(self, key, value):
+        self._data[key.lower()] = value
+
+    def __getitem__(self, key):
+        return self._data[key.lower()]
+
+    def get(self, key, default=None):
+        return self._data.get(key.lower(), default)
+
+
+def read_status_line(stream):
+    line = stream.readline()
+    if not line:
+        raise ValueError("HTTP error: empty status line")
+    parts = line.split(None, 2)
+    if len(parts) < 2:
+        raise ValueError("HTTP error: BadStatusLine:\n%s" % parts)
+    status = int(parts[1])
+    reason = parts[2].rstrip() if len(parts) > 2 else ""
+    return status, reason
+
+
+def read_headers(stream, parse_headers=None):
+    headers = Headers()
+    while True:
+        line = stream.readline()
+        if not line or line == b"\r\n":
+            break
+        if parse_headers is False:
+            continue
+        if parse_headers is True:
+            text = str(line, "utf-8")
+            key, value = text.split(":", 1)
+            headers[key] = value.strip()
+        else:
+            parse_headers(line, headers)
+    return headers
+
+
+def _read_exact(stream, length, max_remaining):
+    out = b""
+    while length > 0:
+        if max_remaining is not None and max_remaining <= 0:
+            raise ValueError("Response body exceeds max_body limit")
+        chunk_len = length
+        if max_remaining is not None and chunk_len > max_remaining:
+            chunk_len = max_remaining
+        chunk = stream.read(chunk_len)
+        if not chunk:
+            break
+        out += chunk
+        length -= len(chunk)
+        if max_remaining is not None:
+            max_remaining -= len(chunk)
+    if length > 0:
+        raise ValueError("Connection closed before Content-Length satisfied")
+    return out, max_remaining
+
+
+def _read_chunked(stream, max_remaining):
+    out = b""
+    while True:
+        line = stream.readline()
+        if not line:
+            raise ValueError("Connection closed in chunked body")
+        size_line = line.strip().split(b";", 1)[0]
+        size = int(size_line, 16)
+        if size == 0:
+            stream.readline()
+            break
+        if max_remaining is not None and size > max_remaining:
+            raise ValueError("Response body exceeds max_body limit")
+        chunk, max_remaining = _read_exact(stream, size, max_remaining)
+        out += chunk
+        stream.readline()
+    return out, max_remaining
+
+
+def _read_until_close(stream, max_remaining):
+    out = b""
+    while True:
+        if max_remaining is not None and max_remaining <= 0:
+            raise ValueError("Response body exceeds max_body limit")
+        to_read = 256
+        if max_remaining is not None:
+            to_read = min(to_read, max_remaining)
+        chunk = stream.read(to_read)
+        if not chunk:
+            break
+        out += chunk
+        if max_remaining is not None:
+            max_remaining -= len(chunk)
+    return out, max_remaining
+
+
+def read_body(stream, headers, max_body=None):
+    """Read the response body according to Content-Length or chunked encoding."""
+    max_remaining = max_body
+    encoding = headers.get("transfer-encoding", "")
+    if encoding:
+        enc = encoding if isinstance(encoding, str) else str(encoding, "utf-8")
+        if "chunked" in enc.lower():
+            body, _ = _read_chunked(stream, max_remaining)
+            return body
+    content_length = headers.get("content-length")
+    if content_length is not None:
+        length = int(content_length)
+        body, _ = _read_exact(stream, length, max_remaining)
+        return body
+    body, _ = _read_until_close(stream, max_remaining)
+    return body
+
+
+def resolve_redirect_url(base_url, location):
+    """Resolve a redirect Location header against the request URL."""
+    loc = location.strip()
+    if loc.startswith("http:") or loc.startswith("https:"):
+        return loc
+    try:
+        proto, dummy, host, path = base_url.split("/", 3)
+    except ValueError:
+        proto, dummy, host = base_url.split("/", 2)
+        path = ""
+    if loc.startswith("/"):
+        return proto + "//" + host + loc
+    if path and "/" in path:
+        dir_path = path.rsplit("/", 1)[0]
+        new_path = dir_path + "/" + loc
+    else:
+        new_path = loc
+    return proto + "//" + host + "/" + new_path