diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 2053b7b276..678ee7d39b 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -22,6 +22,7 @@ PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" VM_BOOT_TIMEOUT=1200 # Overall total boot times are around 15m VM_GREENBOOT_TIMEOUT=1800 # Greenboot readiness may take up to 15-30m depending on the load SKIP_SOS=${SKIP_SOS:-false} # may be overridden in global settings file +SKIP_PCP=${SKIP_PCP:-false} # may be overridden in global settings file SKIP_GREENBOOT=${SKIP_GREENBOOT:-false} # may be overridden in scenario file GREENBOOT_TIMEOUT=${GREENBOOT_TIMEOUT:-600} # may be overridden in scenario file # Container image signature verification should be disabled by default in the @@ -292,6 +293,85 @@ sos_report_for_vm_offline() { "--filename" "*.log" } +collect_pcp_reports_for_vm() { + local -r vmdir="${1}" + local -r vmname="${2}" + + echo "Collecting PCP data from ${vmname}" + run_command_on_vm "${vmname}" "sudo systemctl stop --now pmlogger" || true + + if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then + echo "WARNING: No PCP data directory on ${vmname}, skipping collection" + return 0 + fi + + run_command_on_vm "${vmname}" \ + "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." || true + + mkdir -p "${vmdir}/pcp" + copy_file_from_vm "${vmname}" "/tmp/pcp-archives.tar.gz" "${vmdir}/pcp/" || { + echo "WARNING: Failed to collect PCP data from ${vmname}" + } +} + +collect_pcp_reports_for_vm_offline() { + local -r vmdir="${1}" + local -r vmname="${2}" + local -r full_vmname="$(full_vm_name "${vmname}")" + + echo "Collecting PCP data offline from ${vmname}" + + "${ROOTDIR}/scripts/fetch_tools.sh" "robotframework" + + invoke_qemu_script "wait" \ + "--vm" "${full_vmname}" + + invoke_qemu_script "bash" \ + "--vm" "${full_vmname}" \ + "--args" "sudo systemctl stop --now pmlogger" + + invoke_qemu_script "bash" \ + "--vm" "${full_vmname}" \ + "--args" "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." + + mkdir -p "${vmdir}/pcp" + + invoke_qemu_script "download" \ + "--vm" "${full_vmname}" \ + "--src_dir" "/tmp/" \ + "--dst_dir" "${vmdir}/pcp/" \ + "--filename" "pcp-archives.tar.gz" +} + +collect_pcp_reports() { + if "${SKIP_PCP}"; then + echo "Skipping PCP collection" + return 0 + fi + + echo "Collecting PCP reports" + for vmdir in "${SCENARIO_INFO_DIR}"/"${SCENARIO}"/vms/*; do + if [ ! -d "${vmdir}" ]; then + continue + fi + + local vmname + vmname=$(basename "${vmdir}") + local ip + ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true + + local pcp_func="collect_pcp_reports_for_vm" + if [ -z "${ip}" ]; then + echo "Collecting PCP reports offline" + pcp_func="collect_pcp_reports_for_vm_offline" + fi + + "${pcp_func}" "${vmdir}" "${vmname}" || { + echo "WARNING: Failed to collect PCP data from ${vmname}" + } + done +} + get_lrel_release_image_url() { local -r brew_lrel_release_version="$1" local image_url="" @@ -1526,13 +1606,14 @@ action_create() { fi record_junit "setup" "load_scenario_script" "OK" - # Set the exit handler to attempt the sos report collection and error logging + # Set the exit handler to attempt PCP and SOS report collection and error logging # - Preserve the original exit code # - Log junit message on failure # - Override the exit code if sos report collection fails # shellcheck disable=SC2154 trap 'rc=$? ; \ [ "${rc}" -ne 0 ] && record_junit "setup" "scenario_create_vms" "FAILED" ; \ + collect_pcp_reports || true ; \ sos_report true || rc=1 ; \ close_junit ; exit "${rc}"' EXIT @@ -1584,13 +1665,14 @@ action_run() { fi record_junit "run" "load_scenario_script" "OK" - # Set the exit handler to attempt the sos report collection and error logging + # Set the exit handler to attempt PCP and SOS report collection and error logging # - Preserve the original exit code # - Log junit message on failure # - Override the exit code if sos report collection fails # shellcheck disable=SC2154 trap 'rc=$? ; \ [ "${rc}" -ne 0 ] && record_junit "run" "scenario_run_tests" "FAILED" ; \ + collect_pcp_reports || true ; \ sos_report true || rc=1 ; \ close_junit ; exit "${rc}"' EXIT diff --git a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile index d4bf2dbf96..9441297f95 100644 --- a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile +++ b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile @@ -22,8 +22,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ - systemctl enable microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile index d08f10fb5d..a872170aa7 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile @@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ - systemctl enable microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile index 7ac553d111..6d156f2fdb 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile @@ -22,8 +22,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ - systemctl enable microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile index 982ba1bf45..166d607d24 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile @@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ - systemctl enable microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile index 58efa66116..55224583d0 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile @@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ - systemctl enable microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints/layer1-base/group1/rhel96.toml b/test/image-blueprints/layer1-base/group1/rhel96.toml index 6334d3a567..0b2dc21535 100644 --- a/test/image-blueprints/layer1-base/group1/rhel96.toml +++ b/test/image-blueprints/layer1-base/group1/rhel96.toml @@ -13,8 +13,12 @@ version = "*" name = "iproute-tc" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift-test-agent"] +enabled = ["microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp"] diff --git a/test/image-blueprints/layer1-base/group1/rhel98.toml b/test/image-blueprints/layer1-base/group1/rhel98.toml index 7f9dc37a14..0f004f3828 100644 --- a/test/image-blueprints/layer1-base/group1/rhel98.toml +++ b/test/image-blueprints/layer1-base/group1/rhel98.toml @@ -13,8 +13,12 @@ version = "*" name = "iproute-tc" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift-test-agent"] +enabled = ["microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml index 89e93d4b37..2f2b9d8056 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml @@ -25,8 +25,12 @@ version = "{{ env.Getenv "SOURCE_VERSION_BASE" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml index d16fb10cf9..4ced4d29a5 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml @@ -35,8 +35,12 @@ version = "{{ .Env.FAKE_NEXT_MAJOR_VERSION }}.{{ .Env.FAKE_NEXT_MINOR_VERSION }} name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml index 384606bb8b..64f6097657 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml @@ -39,12 +39,16 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml index 61d2e88732..60bb03c2b7 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml @@ -25,12 +25,16 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml index 42e1729c05..51280fd0c4 100644 --- a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml +++ b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml @@ -15,6 +15,10 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "qemu-guest-agent" version = "*" @@ -34,7 +38,7 @@ name = "skopeo" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent"] +enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml index 4f762059c7..434b6f4f95 100644 --- a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml @@ -39,8 +39,12 @@ version = "{{ env.Getenv "BREW_Y2_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml index 22134c79ab..2bc677ee39 100644 --- a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml @@ -43,8 +43,12 @@ version = "{{ env.Getenv "BREW_Y1_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml index c7353490e7..d35ca71203 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml @@ -43,12 +43,16 @@ version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml index c32156e593..b7d2f8232e 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml @@ -43,12 +43,16 @@ version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/scenario_settings.sh.example b/test/scenario_settings.sh.example index 863ed954a2..391fda56a2 100644 --- a/test/scenario_settings.sh.example +++ b/test/scenario_settings.sh.example @@ -13,6 +13,10 @@ SSH_PUBLIC_KEY=${SSH_PUBLIC_KEY:-${HOME}/.ssh/id_rsa.pub} # local environment where sos can be run manually when necessary. #SKIP_SOS=true +# Disable PCP metrics collection. This can speed up teardown in a +# local environment where PCP can be run manually when necessary. +#SKIP_PCP=true + # Whether to add a VNC graphics console to hosts. This is useful in # local developer settings where cockpit can be used to login to the # host. Set to `true` to enable. Defaults to `false`.