Merge pull request #220 from WebAssembly/http-end-to-end

Add new HTTP end-to-end test

Author: wingo

adapters/wasmtime.py

@@ -63,6 +63,10 @@ def _add_wasi_version_options(argv: List[str], wasi_version: str, proposals: Lis
                 flags_from_proposals += ",http"
             if "sockets" in proposals:
                 flags_from_proposals += ",inherit-network"
+            if "http/service" in proposals:
+                flags_from_proposals += ",cli"
+                argv[splice_pos:splice_pos] = ["serve", "--addr=127.0.0.1:0"]
+                splice_pos += 1
 
             argv[splice_pos:splice_pos] = ["-Wcomponent-model-async",
                                            f"-Sp3{flags_from_proposals}"]

test-runner/requirements/common.txt

@@ -1 +1,2 @@
 colorama>=0.4.3
+requests

test-runner/requirements/dev.txt

@@ -4,3 +4,4 @@ mypy==0.991
 pylint==3.3.8
 pytest==6.2.5
 coverage==6.3.3
+types-requests

test-runner/tests/test_test_case.py

@@ -1,11 +1,14 @@
+import signal
+
 from json import JSONDecodeError
 from pathlib import Path
 from unittest.mock import Mock, patch, mock_open
 
 import pytest
 
 from wasi_test_runner.test_case import (
-    Config, Failure, Result, Run, Wait, Read, Write, Connect, Send, Recv,
+    Config, Failure, Result,
+    Run, Wait, Read, Write, Connect, Send, Recv, Request, Response, Kill,
     ProtocolType, WasiProposal, TestCaseValidator
 )
 
@@ -196,6 +199,23 @@ def test_recv_from_config_with_default_payload() -> None:
     assert recv.payload == ""
 
 
+def test_request_from_config() -> None:
+    req = Request.from_config({"method": "POST", "response": {"body": "hey"}})
+    assert req.method == "POST"
+    assert req.path == "/"
+    assert req.response == Response(status=200, headers={}, body="hey")
+
+
+def test_kill_from_config_with_signal() -> None:
+    kill = Kill.from_config({"signal": "SIGABRT"})
+    assert kill.signal == signal.SIGABRT
+
+
+def test_kill_from_config_with_defaults() -> None:
+    kill = Kill.from_config({})
+    assert kill.signal == signal.SIGTERM
+
+
 @patch(
     "builtins.open",
     new_callable=mock_open,
@@ -304,6 +324,12 @@ def test_dry_run_recv_before_run() -> None:
         validate_config(config)
 
 
+def test_dry_run_request_before_run() -> None:
+    config = Config(operations=[Request.from_config({})])
+    with pytest.raises(AssertionError, match="no process running"):
+        validate_config(config)
+
+
 def test_dry_run_multiple_errors() -> None:
     config = Config(operations=[Read(), Wait(), Run(), Run()])
     with pytest.raises(AssertionError) as exc_info:

test-runner/wasi_test_runner/test_case.py

@@ -1,5 +1,6 @@
 import logging
 import json
+import signal
 from pathlib import Path
 from enum import Enum, StrEnum, auto
 from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Set, Tuple
@@ -10,7 +11,8 @@
 
 
 # Supported operations
-SUPPORTED_OPERATIONS = {"run", "wait", "read", "write", "connect", "send", "recv"}
+SUPPORTED_OPERATIONS = {"run", "wait", "read", "write", "connect",
+                        "send", "recv", "request", "kill"}
 
 
 class WasiVersion(StrEnum):
@@ -166,14 +168,104 @@ def from_config(cls: Type[C], config: Dict[str, Any]) -> C:
         )
 
 
-Operation = Run | Wait | Read | Write | Connect | Send | Recv
+Resp = TypeVar("Resp", bound="Response")
+
+
+class Response(NamedTuple):
+    status: int
+    headers: Dict[str, str]
+    body: str
+
+    @classmethod
+    def from_config(cls: Type[Resp], config: Dict[str, Any]) -> Resp:
+        status = config.get("status", 200)
+        headers = config.get("headers", {})
+        body = config.get("body", "")
+
+        if not isinstance(status, int):
+            raise ValueError("Response status code should be an int")
+        if not isinstance(headers, dict):
+            raise ValueError("Response expected headers should be a dict")
+        for k, v in headers.items():
+            if not isinstance(k, str):
+                raise ValueError("Response expected header name should be a str")
+            if not isinstance(v, str):
+                raise ValueError("Response expected header value should be a str")
+        if not isinstance(body, str):
+            raise ValueError("Response expected body should be a str")
+
+        return cls(status, headers, body)
+
+
+Req = TypeVar("Req", bound="Request")
+
+
+class Request(NamedTuple):
+    method: str
+    path: str
+    response: Response
+
+    @classmethod
+    def from_config(cls: Type[Req], config: Dict[str, Any]) -> Req:
+        method = config.get("method", "GET")
+        path = config.get("path", "/")
+        response = config.get("response", {})
+
+        if not isinstance(method, str):
+            raise ValueError("Request method should be a str")
+        if not isinstance(path, str):
+            raise ValueError("Request path should be a str")
+        response = Response.from_config(response)
+
+        return cls(method, path, response)
+
+
+K = TypeVar("K", bound="Kill")
+
+
+class Kill(NamedTuple):
+    signal: signal.Signals
+
+    @classmethod
+    def from_config(cls: Type[K], config: Dict[str, Any]) -> K:
+        signame = config.get("signal", "SIGTERM")
+
+        if not isinstance(signame, str):
+            raise ValueError(f"Signal name should be a str: {signame}")
+        if signame not in signal.Signals.__members__:
+            raise ValueError(f"Unknown signal name: {signame}")
+
+        return cls(signal.Signals[signame])
+
+
+Operation = Run | Wait | Read | Write | Connect | Send | Recv | Request | Kill
 
 
 class WasiProposal(StrEnum):
     HTTP = 'http'
+    HTTP_SERVICE = 'http/service'
     SOCKETS = 'sockets'
 
 
+def _infer_proposals_from_operations(ops: List[Operation]) -> List[WasiProposal]:
+    sockets = False
+    http_service = False
+    for op in ops:
+        match op:
+            case Recv() | Send() | Connect():
+                sockets = True
+            case Request():
+                http_service = True
+            case _:
+                pass
+    ret = []
+    if sockets:
+        ret.append(WasiProposal.SOCKETS)
+    if http_service:
+        ret.append(WasiProposal.HTTP_SERVICE)
+    return ret
+
+
 T = TypeVar("T", bound="Config")
 
 
@@ -196,8 +288,9 @@ def from_file(cls: Type[T], config_file: str) -> T:
             if dict_config.get("operations") is not None:
                 operations = cls._operations_from_config(test_config_path, dict_config.get("operations"))
 
-            proposals = []
-            if dict_config.get("proposals") is not None:
+            if dict_config.get("proposals") is None:
+                proposals = _infer_proposals_from_operations(operations)
+            else:
                 proposals = cls._proposals_from_config(dict_config.get("proposals"))
 
             return cls(operations=operations, proposals=proposals)
@@ -276,6 +369,10 @@ def _operations_from_config(cls: Type[T], test_config_path: Path, ops: List[Any]
                     operations.append(Send.from_config(op))
                 case "recv":
                     operations.append(Recv.from_config(op))
+                case "request":
+                    operations.append(Request.from_config(op))
+                case "kill":
+                    operations.append(Kill.from_config(op))
 
         return operations
 
@@ -321,6 +418,12 @@ def do_send(self, send: Send) -> None:
     def do_recv(self, recv: Recv) -> None:
         raise NotImplementedError()
 
+    def do_request(self, req: Request) -> None:
+        raise NotImplementedError()
+
+    def do_kill(self, kill: Kill) -> None:
+        raise NotImplementedError()
+
     def do_cleanup(self, successful: bool) -> None:
         raise NotImplementedError()
 
@@ -345,27 +448,33 @@ def run(self) -> Result:
                 #   wasi_test_runner/runtime_adapter.py:131: error: Argument 2 to "_handle_read"
                 #   has incompatible type "Read"; expected "Read" [arg-type]
                 match op:
-                    case Run() as run:
-                        assert isinstance(run, Run)
-                        self.do_run(run)
-                    case Write() as write:
-                        assert isinstance(write, Write)
-                        self.do_write(write)
-                    case Read() as read:
-                        assert isinstance(read, Read)
-                        self.do_read(read)
-                    case Wait() as wait:
-                        assert isinstance(wait, Wait)
-                        self.do_wait(wait)
-                    case Connect() as conn:
-                        assert isinstance(conn, Connect)
-                        self.do_connect(conn)
-                    case Send() as send:
-                        assert isinstance(send, Send)
-                        self.do_send(send)
-                    case Recv() as recv:
-                        assert isinstance(recv, Recv)
-                        self.do_recv(recv)
+                    case Run():
+                        assert isinstance(op, Run)
+                        self.do_run(op)
+                    case Write():
+                        assert isinstance(op, Write)
+                        self.do_write(op)
+                    case Read():
+                        assert isinstance(op, Read)
+                        self.do_read(op)
+                    case Wait():
+                        assert isinstance(op, Wait)
+                        self.do_wait(op)
+                    case Connect():
+                        assert isinstance(op, Connect)
+                        self.do_connect(op)
+                    case Send():
+                        assert isinstance(op, Send)
+                        self.do_send(op)
+                    case Recv():
+                        assert isinstance(op, Recv)
+                        self.do_recv(op)
+                    case Request():
+                        assert isinstance(op, Request)
+                        self.do_request(op)
+                    case Kill():
+                        assert isinstance(op, Kill)
+                        self.do_kill(op)
 
             successful = not self.has_failure()
         finally:
@@ -445,6 +554,12 @@ def do_recv(self, recv: Recv) -> None:
         self.assert_proc(recv)
         self.assert_stream(recv, recv.id, StreamType.SOCKET)
 
+    def do_request(self, req: Request) -> None:
+        self.assert_proc(req)
+
+    def do_kill(self, kill: Kill) -> None:
+        self.assert_proc(kill)
+
     def do_cleanup(self, successful: bool) -> None:
         if successful:
             self.assert_no_proc(self._config_path)

test-runner/wasi_test_runner/test_suite_runner.py

@@ -11,13 +11,15 @@
 from pathlib import Path
 from typing import List, NamedTuple, Tuple, Dict, Any, IO
 
+import requests
+
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
     Result, Failure, WasiVersion, Config,
     TestCase, TestCaseRunnerBase, TestCaseValidator,
     # Operation types
-    Run, Read, Write, Wait, Send, Recv, Connect
+    Run, Read, Write, Wait, Send, Recv, Connect, Request, Kill
 )
 from .reporters import TestReporter
 from .test_suite import TestSuite, TestSuiteMeta
@@ -38,6 +40,7 @@ class TestCaseRunner(TestCaseRunnerBase):
     _pipes: Dict[str, IO[str]]
     _sockets: Dict[str, socket.socket]
     _last_argv: List[str]
+    _http_server: str | None
 
     def __init__(self, config: Config, test_path: str, wasi_version: WasiVersion,
                  runtime: RuntimeAdapter) -> None:
@@ -50,6 +53,7 @@ def __init__(self, config: Config, test_path: str, wasi_version: WasiVersion,
         self._pipes = {}
         self._sockets = {}
         self._last_argv = []
+        self._http_server = None
 
     def _add_cleanup_dir(self, d: Path) -> None:
         _cleanup_test_output(d)
@@ -88,6 +92,18 @@ def get_pipe(self, name: str) -> IO[str]:
     def last_argv(self) -> List[str]:
         return self._last_argv
 
+    def get_http_server(self) -> str | None:
+        if self._http_server:
+            return self._http_server
+        line = self.get_pipe('stderr').readline().strip()
+        start = line.find('http://')
+        if start < 0:
+            self.fail_unexpected(f"Expected 'http://' in first line, got {line}")  # noqa: E231
+            return None
+        # The server URL starts with http:// and ends at EOL or whitespace.
+        self._http_server = line[start:].split()[0]
+        return self._http_server
+
     def do_run(self, run: Run) -> None:
         for (host, _guest) in run.dirs:
             self._add_cleanup_dir(host)
@@ -178,6 +194,46 @@ def do_recv(self, recv: Recv) -> None:
         except UnicodeDecodeError as e:
             self.fail_unexpected(f"{recv}: Failed to decode response: {e}")
 
+    def do_request(self, req: Request) -> None:
+        # pylint: disable-msg=too-many-return-statements
+        http_server = self.get_http_server()
+        if http_server is None:
+            return
+        url = http_server + req.path
+        try:
+            response = requests.request(req.method, url, timeout=5)
+        except requests.exceptions.Timeout:
+            self.fail_unexpected(f"{req}: Timeout waiting for response")
+            return
+        except requests.exceptions.RequestException as e:
+            self.fail_unexpected(f"{req}: Failed to make request: {e}")
+            return
+        if response.status_code != req.response.status:
+            self.fail_unexpected(
+                f"{req}: Expected status {req.response.status}, got {response.status_code}")
+            return
+        for h, expected in req.response.headers.items():
+            if h not in response.headers:
+                self.fail_unexpected(f"{req}: Response missing header {h}")
+                return
+            actual = response.headers[h]
+            if actual != expected:
+                self.fail_unexpected(
+                    f"{req}: Expected response header {h}={expected}, got {actual}")
+                return
+        if response.text != req.response.body:
+            self.fail_unexpected(
+                f"{req}: Expected response body '{req.response.body}', got '{response.text}'")
+            return
+
+    def do_kill(self, kill: Kill) -> None:
+        try:
+            proc = self._proc
+            assert proc is not None
+            proc.send_signal(kill.signal)
+        except OSError as e:
+            self.fail_unexpected(f"{kill}: Failed to send {kill.signal}: {e}")
+
     def do_cleanup(self, successful: bool) -> None:
         if self._proc:
             self._proc.kill()