By default, skip tests for unsupported WASI worlds

Author: wingo

test-runner/tests/test_test_suite.py

@@ -21,7 +21,8 @@ def test_test_suite_should_return_correct_count() -> None:
         ts.TestSuiteMeta("suite",
                          tc.WasiVersion.WASM32_WASIP1,
                          RuntimeMeta("test-runtime", "3.14",
-                                     frozenset([tc.WasiVersion.WASM32_WASIP1]))),
+                                     frozenset([tc.WasiVersion.WASM32_WASIP1]),
+                                     frozenset([tc.WasiWorld.CLI_COMMAND]))),
         10.0,
         datetime.now(),
         [

test-runner/tests/test_test_suite_runner.py

@@ -78,8 +78,10 @@ def test_runner_end_to_end() -> None:
     runtime_version_str = "4.2"
     the_runtime_wasi_version = tc.WasiVersion.WASM32_WASIP1
     runtime_wasi_versions = frozenset([the_runtime_wasi_version])
+    runtime_wasi_worlds = frozenset([tc.WasiWorld.CLI_COMMAND])
     runtime_meta = RuntimeMeta(runtime_name, runtime_version_str,
-                               runtime_wasi_versions)
+                               runtime_wasi_versions,
+                               runtime_wasi_worlds)
 
     expected_test_suite_meta = ts.TestSuiteMeta(test_suite_name,
                                                 the_runtime_wasi_version,
@@ -139,7 +141,8 @@ def test_runner_end_to_end() -> None:
         assert filt.should_skip.call_count == 4
         for test_case in expected_test_cases:
             filt.should_skip.assert_any_call(expected_test_suite_meta,
-                                             test_case.name)
+                                             test_case.name,
+                                             test_case.config)
 
 
 @patch("os.path.exists", Mock(return_value=False))

test-runner/wasi_test_runner/filters.py

@@ -5,22 +5,25 @@
 import json
 
 from .test_suite import TestSuiteMeta
+from .test_case import Config
 
 
 class TestFilter(ABC):
     @abstractmethod
     def should_skip(
-        self, meta: TestSuiteMeta, test_name: str
+        self, meta: TestSuiteMeta, test_name: str, config: Config
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         pass
 
 
 class UnsupportedWasiTestExcludeFilter(TestFilter):
     def should_skip(
-        self, meta: TestSuiteMeta, test_name: str
+        self, meta: TestSuiteMeta, test_name: str, config: Config
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         if meta.wasi_version not in meta.runtime.supported_wasi_versions:
             return True, "WASI version unsupported by runtime"
+        if config.world not in meta.runtime.supported_wasi_worlds:
+            return True, "WASI world unsupported by runtime"
         return False, None
 
 
@@ -30,7 +33,7 @@ def __init__(self, filename: str) -> None:
             self.filter_dict = json.load(file)
 
     def should_skip(
-        self, meta: TestSuiteMeta, test_name: str
+        self, meta: TestSuiteMeta, test_name: str, config: Config
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         test_suite_filter = self.filter_dict.get(meta.name)
         if test_suite_filter is None:

test-runner/wasi_test_runner/runtime_adapter.py

@@ -11,6 +11,7 @@ class RuntimeMeta(NamedTuple):
     name: str
     version: str
     supported_wasi_versions: frozenset[WasiVersion]
+    supported_wasi_worlds: frozenset[WasiWorld]
 
     def __str__(self) -> str:
         return f"{self.name} {self.version}"
@@ -83,11 +84,14 @@ def __init__(self, adapter_path: str) -> None:
             wasi_versions = frozenset(
                 WasiVersion(v) for v in self._adapter.get_wasi_versions()
             )
+            wasi_worlds = frozenset(
+                WasiWorld(w) for w in self._adapter.get_wasi_worlds()
+            )
         except subprocess.CalledProcessError as e:
             raise UnavailableRuntimeAdapterError(adapter_path, e) from e
         except FileNotFoundError as e:
             raise UnavailableRuntimeAdapterError(adapter_path, e) from e
-        self._meta = RuntimeMeta(name, version, wasi_versions)
+        self._meta = RuntimeMeta(name, version, wasi_versions, wasi_worlds)
 
     def get_meta(self) -> RuntimeMeta:
         return self._meta

test-runner/wasi_test_runner/test_suite_runner.py

@@ -265,17 +265,24 @@ def run_tests_from_test_suite(
     meta = TestSuiteMeta(manifest.name, manifest.wasi_version,
                          runtime.get_meta())
 
-    for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
-        test_name = os.path.splitext(os.path.basename(test_path))[0]
+    all_tests = [
+        (test_path,
+         os.path.splitext(os.path.basename(test_path))[0],
+         _read_test_config(test_path))
+        for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm"))
+    ]
+
+    for test_path, name, config in all_tests:
         for filt in filters:
             # for now, just drop the skip reason string. it might be
             # useful to make reporters report it.
-            skip, _ = filt.should_skip(meta, test_name)
+            skip, _ = filt.should_skip(meta, name, config)
             if skip:
-                test_case = _skip_single_test(test_path)
+                test_case = _skip_single_test(name, config)
                 break
         else:
-            test_case = _execute_single_test(runtime, meta, test_path)
+            test_case = _execute_single_test(
+                runtime, meta, test_path, name, config)
         test_cases.append(test_case)
         for reporter in reporters:
             reporter.report_test(meta, test_case)
@@ -290,10 +297,9 @@ def run_tests_from_test_suite(
     )
 
 
-def _skip_single_test(test_path: str) -> TestCase:
-    config = _read_test_config(test_path)
+def _skip_single_test(name: str, config: Config) -> TestCase:
     return TestCase(
-        name=os.path.splitext(os.path.basename(test_path))[0],
+        name=name,
         argv=[],
         config=config,
         result=Result(is_executed=False, failures=[]),
@@ -320,16 +326,16 @@ def _cleanup_test_output(host_dir: Path) -> None:
 
 
 def _execute_single_test(
-    runtime: RuntimeAdapter, meta: TestSuiteMeta, test_path: str
+        runtime: RuntimeAdapter, meta: TestSuiteMeta, test_path: str,
+        name: str, config: Config
 ) -> TestCase:
-    config = _read_test_config(test_path)
     runner = TestCaseRunner(config, test_path, meta.wasi_version, runtime)
     test_start = time.time()
     result = runner.run()
     elapsed = time.time() - test_start
 
     return TestCase(
-        name=os.path.splitext(os.path.basename(test_path))[0],
+        name=name,
         argv=runner.last_argv(),
         config=config,
         result=result,