Merge branch '3.x' into feat/ci-cd-release-improvements

Author: wowi42

.editorconfig

@@ -1,3 +1,12 @@
+[*]
+end_of_line = lf
+insert_final_newline = true
+max_line_length = 100
+
+[*.py]
+indent_style = space
+indent_size = 4
+
 [*.json]
 indent_style = space
 indent_size = 4

.gitignore

@@ -31,6 +31,7 @@ docs/_deploy_globals.rst
 
 .idea/
 .vscode/
+.claude/settings.local.json
 
 pyinfra-debug.log
 Makefile

AGENTS.md

@@ -0,0 +1,155 @@
+# AGENTS.md
+
+This file provides guidance to coding agents, like Claude Code (claude.ai/code), when working with 
+code in this repository.
+
+## About pyinfra
+
+pyinfra turns Python code into shell commands and runs them on servers. Think Ansible, but Python
+instead of YAML, and much faster. It supports SSH, local machine, Docker, and more via connectors.
+
+## Development Setup
+
+```bash
+uv sync  # Install all dependencies into managed venv
+```
+
+## Common Commands
+
+```bash
+# Run tests
+scripts/dev-test.sh
+# or directly:
+uv run pytest --cov --disable-warnings -m 'not end_to_end'
+
+# Run fixtures for a single operation or fact
+uv run pytest tests/test_operations.py -k "apt.packages"
+uv run pytest tests/test_facts.py -k "LinuxHardware"
+
+# End-to-end tests (require Docker/SSH/local targets)
+uv run pytest -m end_to_end_local
+uv run pytest -m end_to_end_docker
+
+# Lint and type check
+scripts/dev-lint.sh
+# Individually:
+uv run ruff check
+uv run ruff format --check
+uv run mypy
+uv run python scripts/lint_arguments_sync.py
+
+# Auto-format
+scripts/dev-format.sh
+```
+
+## Architecture
+
+The repo has two top-level packages under `src/`:
+
+- **`pyinfra/`** — core library (operations, facts, connectors, API)
+- **`pyinfra_cli/`** — CLI wrapper using Click + gevent
+
+### Core Concepts
+
+**Operations** (`src/pyinfra/operations/`) — Declarative functions (e.g. `apt.packages`,
+`files.put`) that describe desired state. Each operation uses the `@operation` decorator from
+`api/operation.py`, generates a list of commands, and is idempotent. Operations call facts to
+read current state, then return commands to reach desired state.
+
+**Facts** (`src/pyinfra/facts/`) — Read system state (e.g. `AptPackages`, `LinuxHardware`). Each
+fact is a class extending `FactBase` with a `command` attribute and a `process()` method that
+parses command output. Facts are cached per-host per-run.
+
+**Connectors** (`src/pyinfra/connectors/`) — Abstractions for how to connect to and execute on a
+target (SSH, local, Docker, chroot, Terraform, Vagrant, etc.). New connectors should be separate
+packages, not added to this repo.
+
+**API** (`src/pyinfra/api/`) — The core engine:
+- `state.py` — Global deploy state, callbacks, host grouping
+- `host.py` — Per-host metadata and fact access
+- `inventory.py` — Collection of hosts and groups
+- `operation.py` — `@operation` decorator that wraps functions into deploy operations
+- `operations.py` — Execution engine that runs operations across hosts
+- `command.py` — Command types: `StringCommand`, `FileUploadCommand`, `RsyncCommand`,
+  `QuoteString`, `MaskString`, etc.
+- `facts.py` — Fact base classes and execution logic
+- `deploy.py` — `@deploy` decorator for grouping operations
+- `connect.py` — Connector lifecycle (connect/disconnect)
+- `config.py` — `Config` object with all configuration options
+- `arguments.py` / `arguments_typed.py` — Global operation arguments (e.g. `_sudo`, `_su_user`);
+  **these two files must stay in sync** — CI enforces this via `scripts/lint_arguments_sync.py`,
+  so touching one requires touching the other
+- `output.py` — Pluggable output functions (decoupled from Click for testability)
+
+**Context** (`src/pyinfra/context.py`) — Thread-local (gevent-safe) context objects: `host`,
+`state`, `config`, `inventory`. Operations access the current host via `pyinfra.context.host`
+rather than explicit passing.
+
+**Concurrency** — Uses gevent greenlets for parallel host execution. `pyinfra_cli/main.py`
+monkey-patches stdlib at startup.
+
+### Adding Operations / Facts
+
+Operations and facts are auto-discovered from their respective directories. A new
+`src/pyinfra/operations/mytool.py` is immediately available as `from pyinfra.operations import
+mytool`.
+
+- Operations must be idempotent and use facts to check current state
+- Facts must implement `command` (shell command to run) and `process(output)` (parse result)
+- Both need corresponding tests (see fixture convention below)
+- Every operation/fact module must be registered in `pyinfra-metadata.toml` as a plugin with
+  tags; omitting this won't break tests but will break docs generation
+
+**Operation / fact tests are YAML or JSON fixtures, not Python tests.** Drop a file under
+`tests/operations/<module>.<op>/` or `tests/facts/<module>.<Fact>/` — it is auto-discovered by
+the `testgen` metaclass. Prefer YAML for new fixtures. To cover a new code path, add a fixture —
+do not write a new Python test.
+
+Operation fixture structure (`tests/operations/<module>.<op>/<name>.yaml`):
+
+```yaml
+args:
+  - positional_arg
+kwargs:
+  param: value
+facts:
+  module.FactClass: {}          # a dict of mock values keyed by object_id and attribute
+commands:
+  - shell command that should be produced
+```
+
+Optional keys: `exception` (e.g. `{name: OperationError, message: "..."}`), `noop_description`.
+
+Fact fixture structure (`tests/facts/<module>.<Fact>/<name>.yaml`):
+
+```yaml
+command: shell command the fact runs
+requires_command: binary               # optional
+output: |
+  raw stdout to parse
+fact:
+  item:
+    key: value                         # expected return value of process()
+```
+
+**Docstring format** — pyinfra uses `+ param: description` bullets (parsed by
+`scripts/generate_operations_docs.py`). Do not use Google/NumPy/Sphinx style — it will silently
+break docs generation.
+
+**Shell safety** — user-supplied values must be composed into shell commands using `StringCommand`
++ `QuoteString` / `MaskString` from `pyinfra.api`. Do not use plain string formatting (e.g.
+`"rm -f {}".format(path)`) for user-controlled values.
+
+**Optional parameter defaults** — optional parameters must default to `None`, not `""`. Older
+operations in the codebase use `""` defaults; do not replicate this pattern.
+
+## Branch Strategy
+
+PRs target the latest major branch (`3.x`). One branch per major version exists (`2.x`, `1.x`,
+etc.).
+
+## PR Checklist
+
+- Tests pass (`scripts/dev-test.sh`)
+- Lint/types pass (`scripts/dev-lint.sh`)
+- New operations/facts include tests and documentation

AI_POLICY.md

@@ -0,0 +1,48 @@
+# AI Usage Policy
+
+The pyinfra project has strict rules for AI usage:
+
+- **All AI usage in any form must be disclosed.** You must state the tool you used (e.g. Claude
+  Code, Cursor, Amp) along with the extent that the work was AI-assisted.
+
+- **The human-in-the-loop must fully understand all code.** If you can't explain what your changes
+  do and how they interact with the greater system without the aid of AI tools, do not contribute
+  to this project.
+
+- **Issues and discussions can use AI assistance but must have a full human-in-the-loop.** This
+  means that any content generated with AI must have been reviewed _and edited_ by a human before
+  submission. AI is very good at being overly verbose and including noise that distracts from the
+  main point. Humans must do their research and trim this down.
+
+- **No AI-generated media is allowed (art, images, videos, audio, etc.).** Text and code are the
+  only acceptable AI-generated content, per the other rules in this policy.
+
+## There are Humans Here
+
+Please remember that pyinfra is maintained by humans.
+
+Every discussion, issue, and pull request is read and reviewed by humans (and sometimes machines,
+too). It is a boundary point at which people interact with each other and the work done. It is
+rude and disrespectful to approach this boundary with low-effort, unqualified work, since it puts
+the burden of validation on the maintainer.
+
+In a perfect world, AI would produce high-quality, accurate work every time. But today, that
+reality depends on the driver of the AI. And today, most drivers of AI are just not good enough.
+So, until either the people get better, the AI gets better, or both, we have to have strict rules
+to protect maintainers.
+
+## AI is Welcome Here
+
+pyinfra is written with plenty of AI assistance, and many maintainers embrace AI tools as a
+productive tool in their workflow. As a project, we welcome AI as a tool!
+
+**Our reason for the strict AI policy is not due to an anti-AI stance**, but instead due to the
+number of highly unqualified people using AI. It's the people, not the tools, that are the problem.
+
+I include this section to be transparent about the project's usage about AI for people who may
+disagree with it, and to address the misconception that this policy is anti-AI in nature.
+
+## Attribution
+
+This policy is based on the AI Usage Policy of the
+[Ghostty project](https://github.com/ghostty-org/ghostty/blob/main/AI_POLICY.md).

CHANGELOG.md

@@ -1,3 +1,16 @@
+# Unreleased
+
+Core:
+
+- api.facts: `requires_command` now uses an if/then/else shell guard with a sentinel marker
+  (`##PYINFRA_NOCMD##`) so that "binary absent" can be distinguished from "binary returned no data"
+- api.facts: add `check_preconditions(state, host)` hook on `FactBase` for runtime prerequisite checks
+  (e.g. kernel module loaded, service running) — return a reason string or `None`
+- api.exceptions: add `FactNotCollected`, `MissingCommandError`, `FactPreconditionError` exception
+  hierarchy; both exceptions are phase-aware (silent during prepare, raised during execute)
+- facts.zfs: fix `ZfsDatasets.requires_command` returning `"zpool"` instead of `"zfs"`; add
+  `ZfsPools` fact; add `ZfsDatasets.check_preconditions()` checking kernel module via `server.KernelModules`
+
 # v3.7
 
 Thank you to all contributors - particular shout out to @wowi42 for an incredible run of PRs!

CLAUDE.md

@@ -0,0 +1,3 @@
+# CLAUDE.md
+
+See @AGENTS.md for guidance.

docs/api/facts.md

@@ -5,9 +5,80 @@ and a ``process`` function. The command is executed on the target host and the o
 passed (as a ``list`` of lines) to the ``process`` handler to generate fact data. Facts can output anything, normally a ``list`` or ``dict``.
 
 Fact classes may provide a ``default`` function that takes no arguments (except ``self``). The return value of this function is used if an error
-occurs during fact collection. Additionally, a ``requires_command`` variable can be set on the fact that specifies a command that must be available
-on the host to collect the fact. If this command is not present on the host, the fact will be set to the default, or empty if no ``default`` function
-is available.
+occurs during fact collection.
+
+## Guarding against missing binaries: `requires_command`
+
+Override `requires_command` to declare a binary that must be present on the remote host before
+the fact command is run. When the binary is absent pyinfra emits a unique sentinel instead of
+executing the command, then raises `MissingCommandError` internally:
+
+```py
+from pyinfra.api import FactBase
+
+class ZfsPools(FactBase):
+    def requires_command(self) -> str:
+        return "zpool"
+
+    def command(self) -> str:
+        return "zpool get -H all"
+```
+
+**Phase-aware behaviour** — The exception is handled differently depending on the deploy phase:
+
+- **Prepare phase**: the fact returns `default()` silently. The binary may simply not be
+  installed yet; a later operation will install it.
+- **Execute phase** (v3): a warning is logged and `default()` is returned. This preserves
+  backwards compatibility for deploys that rely on `default()` being returned when a binary
+  is absent.
+- **Execute phase** (v4): `MissingCommandError` will be raised so the developer knows the
+  deploy is incorrectly ordered (the install step must come first).
+
+## Checking runtime prerequisites: `check_preconditions()`
+
+Some facts require more than just a binary — they need a specific runtime state (e.g. a kernel
+module loaded, a service running). Override `check_preconditions` to express these checks:
+
+```py
+from pyinfra.api import FactBase
+
+class ZfsDatasets(FactBase):
+    def requires_command(self) -> str:
+        return "zfs"
+
+    def check_preconditions(self, state, host):
+        from pyinfra.facts.server import KernelModules
+        modules = host.get_fact(KernelModules) or {}
+        if "zfs" not in modules:
+            return "kernel module 'zfs' is not loaded"
+
+    def command(self) -> str:
+        return "zfs get -H all"
+```
+
+Return values:
+
+| Return value | Meaning |
+|---|---|
+| `None` (or no return) | Prerequisites satisfied — proceed normally |
+| `"reason"` | Prerequisite not satisfied with a human-readable explanation |
+
+The framework raises `FactPreconditionError` automatically and applies the same
+phase-aware behaviour as `requires_command`: silent during prepare, raised during execute.
+Fact authors never need to import any exception class.
+
+## Exception hierarchy
+
+All "fact skipped" situations use a common base class so callers can catch at any level:
+
+```
+FactError
+└── FactNotCollected          # base: fact could not be collected
+    ├── MissingCommandError   # requires_command binary absent
+    └── FactPreconditionError # check_preconditions() not satisfied
+```
+
+All three are exported from `pyinfra.api`.
 
 ## Importing & Using Facts
 

scripts/generate_facts_docs.py

@@ -104,6 +104,13 @@ def build_facts_docs():
                     )
 
             lines.append(".. _facts:{0}.{1}:".format(module_name, name))
+            # Modules that re-export classes under an alias (e.g. facts/zfs.py
+            # exposes both ZfsDatasets and Datasets pointing at the same class)
+            # end up keyed by whichever name getmembers sees first. Emit the
+            # canonical class name as an extra label so cross-refs from the
+            # operations docs resolve regardless of import style.
+            if cls.__name__ != name:
+                lines.append(".. _facts:{0}.{1}:".format(module_name, cls.__name__))
             lines.append("")
 
             title = ":code:`{0}.{1}`".format(module_name, name)

src/pyinfra/api/__init__.py

@@ -11,11 +11,14 @@
 from .deploy import deploy  # noqa: F401 # pragma: no cover
 from .exceptions import (  # noqa: F401
     DeployError,  # noqa: F401 # pragma: no cover
+    FactPreconditionError,
     FactError,
+    FactNotCollected,
     FactProcessError,
     FactTypeError,
     FactValueError,
     InventoryError,
+    MissingCommandError,
     OperationError,
     OperationTypeError,
     OperationValueError,

src/pyinfra/api/exceptions.py

@@ -60,6 +60,38 @@ class NestedOperationError(OperationError):
     """
 
 
+class FactNotCollected(FactError):
+    """
+    Base exception raised when a fact could not be collected (e.g. binary absent
+    on the remote host, or the fact was skipped by a condition).
+    """
+
+
+class MissingCommandError(FactNotCollected):
+    """
+    Exception raised when ``requires_command`` specifies a binary that is
+    not present on the remote host.  The fact returns its ``default()`` value
+    instead of raising, unless explicitly configured otherwise.
+    """
+
+    def __init__(self, command: str) -> None:
+        self.command = command
+        super().__init__(f"Command not found on remote host: {command}")
+
+
+class FactPreconditionError(FactNotCollected):
+    """
+    Exception raised when a fact's ``check_preconditions()`` returns a reason string
+    (e.g. a kernel module is not loaded).  Like ``MissingCommandError``, this is
+    silenced during the prepare phase and re-raised during execute.
+    """
+
+    def __init__(self, fact_cls: type, reason: str) -> None:
+        self.fact_cls = fact_cls
+        self.reason = reason
+        super().__init__(f"Fact precondition not satisfied ({fact_cls.__name__}): {reason}")
+
+
 class DeployError(PyinfraError):
     """
     User exception for raising in deploys or sub deploys.