Fixes #353 - Add health check command

Add a new foremanctl health command that verifies the state of all
Foreman services after installation or during troubleshooting.

Checks performed:
- Core services running (foreman, httpd, redis, postgresql)
- Dynflow workers running (orchestrator, worker, worker-hosts-queue)
- Pulp services running (pulp-api, pulp-content)
- Candlepin service running
- Foreman API responding (GET /api/v2/ping)
- Foreman tasks status (via Katello ping response)

Reports a summary of all failures and exits non-zero if any check
fails, making it suitable for scripting and CI use.

Author: jakduch

.github/workflows/test.yml

@@ -120,6 +120,9 @@ jobs:
       - name: Add optional features - azure-rm, google and remote-execution
         run: |
           ./foremanctl deploy --add-feature azure-rm --add-feature google --add-feature remote-execution
+      - name: Run health check
+        run: |
+          ./foremanctl health
       - name: Run tests
         run: |
           ./forge test --pytest-args="--certificate-source=${{ matrix.certificate_source }} --database-mode=${{ matrix.database }}"
@@ -241,6 +244,9 @@ jobs:
       - name: Run deployment
         run: |
           ./foremanctl deploy
+      - name: Run health check
+        run: |
+          ./foremanctl health
       - name: Run tests
         run: |
           ./forge test

src/filter_plugins/foremanctl.py

@@ -94,6 +94,10 @@ def available_foreman_proxy_plugins(_value):
     return compact_list(plugins)
 
 
+def has_feature(features, feature):
+    """Check if a feature is enabled - exact match or prefix (feature/)."""
+    return feature in features or any(f.startswith(feature + '/') for f in features)
+
 class FilterModule(object):
     '''foremanctl filters'''
 
@@ -106,4 +110,5 @@ def filters(self):
             'available_foreman_proxy_plugins': available_foreman_proxy_plugins,
             'list_all_features': list_all_features,
             'invalid_features': invalid_features,
+            'has_feature': has_feature,
         }

src/playbooks/health/health.yaml

@@ -0,0 +1,22 @@
+---
+- name: Run Foreman health checks
+  hosts: quadlet
+  become: true
+  gather_facts: true
+  vars_files:
+    - "../../vars/defaults.yml"
+    - "../../vars/flavors/{{ flavor }}.yml"
+    - "../../vars/base.yaml"
+  tasks:
+    - name: Execute health checks
+      ansible.builtin.include_role:
+        name: checks
+        tasks_from: execute_check
+      loop:
+        - check_services
+        - check_foreman_api
+    - name: Report status of health checks
+      ansible.builtin.fail:
+        msg: "{{ checks_results }}"
+      when:
+        - checks_results|default([])|length > 0

src/playbooks/health/metadata.obsah.yaml

@@ -0,0 +1,5 @@
+---
+help: |
+  Run health checks on Foreman services
+include:
+  - _database_mode

src/roles/check_foreman_api/tasks/main.yaml

@@ -0,0 +1,18 @@
+---
+- name: Check Foreman API responds
+  ansible.builtin.uri:
+    url: "{{ foreman_url }}/api/v2/ping"
+    validate_certs: false
+    status_code: 200
+    timeout: 10
+  register: check_foreman_api_ping
+
+- name: Check Foreman tasks status
+  ansible.builtin.assert:
+    that:
+      - check_foreman_api_ping.json.results.katello.services.foreman_tasks.status == 'ok'
+    fail_msg: "Foreman tasks status: {{ check_foreman_api_ping.json.results.katello.services.foreman_tasks.status | default('unknown') }}"
+    success_msg: "Foreman tasks status: ok"
+  when:
+    - enabled_features | has_feature('content')
+    - check_foreman_api_ping.json.results.katello is defined

src/roles/check_services/tasks/main.yaml

@@ -0,0 +1,66 @@
+---
+- name: Gather service facts
+  ansible.builtin.service_facts:
+
+- name: Check core services are running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services[item + '.service'] is defined"
+      - "ansible_facts.services[item + '.service']['state'] == 'running'"
+    fail_msg: "Service {{ item }} is not running"
+    success_msg: "Service {{ item }} is running"
+  loop:
+    - foreman
+    - httpd
+    - redis
+
+- name: Check PostgreSQL service is running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services['postgresql.service'] is defined"
+      - "ansible_facts.services['postgresql.service']['state'] == 'running'"
+    fail_msg: "Service postgresql is not running"
+    success_msg: "Service postgresql is running"
+  when: database_mode | default('internal') == 'internal'
+
+- name: Check dynflow services are running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services[item + '.service'] is defined"
+      - "ansible_facts.services[item + '.service']['state'] == 'running'"
+    fail_msg: "Service {{ item }} is not running"
+    success_msg: "Service {{ item }} is running"
+  loop:
+    - dynflow-sidekiq@orchestrator
+    - dynflow-sidekiq@worker
+    - dynflow-sidekiq@worker-hosts-queue
+
+- name: Check Pulp services are running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services[item + '.service'] is defined"
+      - "ansible_facts.services[item + '.service']['state'] == 'running'"
+    fail_msg: "Service {{ item }} is not running"
+    success_msg: "Service {{ item }} is running"
+  loop:
+    - pulp-api
+    - pulp-content
+  when: enabled_features | has_feature('content')
+
+- name: Check Candlepin service is running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services['candlepin.service'] is defined"
+      - "ansible_facts.services['candlepin.service']['state'] == 'running'"
+    fail_msg: "Service candlepin is not running"
+    success_msg: "Service candlepin is running"
+  when: enabled_features | has_feature('content')
+
+- name: Check Foreman Proxy service is running
+  ansible.builtin.assert:
+    that:
+      - "ansible_facts.services['foreman-proxy.service'] is defined"
+      - "ansible_facts.services['foreman-proxy.service']['state'] == 'running'"
+    fail_msg: "Service foreman-proxy is not running"
+    success_msg: "Service foreman-proxy is running"
+  when: enabled_features | has_feature('foreman-proxy')