From fe97eaf960f5d8f82297ce966e7efc75897e5e91 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Thu, 21 May 2026 18:21:26 +0200 Subject: [PATCH 01/20] SSI:Test dd agent --- .../provisions/auto-inject/auto-inject_installer_manual.yml | 6 +++--- .../auto-inject/docker/docker-compose-agent-prod.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index d74b3c540d1..937bf0589cc 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -9,9 +9,9 @@ local_path: binaries/ remote-command: | - # Pin to 7.78.4 agent release. APMSP-3059 - export DD_AGENT_MAJOR_VERSION=7 - export DD_AGENT_MINOR_VERSION=78.4 + # APMSP-3059 + export DD_INSTALLER_AGENT_VERSION=114350808 + export DD_INSTALLER_INSTALLER_VERSION=114350808 # Check if Docker is installed and ensure it's running if command -v docker >/dev/null 2>&1; then echo "Docker is installed, ensuring service is enabled and running..." diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index ee6c3fe3d54..61ef05eba14 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -2,7 +2,7 @@ services: datadog: container_name: dd-agent # Pin to 7.78.4 agent release. APMSP-3059 - image: gcr.io/datadoghq/agent:7.78.4 + image: registry.ddbuild.io/datadog-agent:dev-gabedos-revert-d01d852-apm-container-tags-275edf32-jmx-zstd environment: - DD_API_KEY=${DD_API_KEY} - DD_SITE=datadoghq.com From de7cd7b67d657f4ae911f9fe675b216fb3f6f957 Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Thu, 21 May 2026 14:32:40 -0400 Subject: [PATCH 02/20] use docker public image --- .../provisions/auto-inject/docker/docker-compose-agent-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index 61ef05eba14..5d4f269bcea 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -2,7 +2,7 @@ services: datadog: container_name: dd-agent # Pin to 7.78.4 agent release. APMSP-3059 - image: registry.ddbuild.io/datadog-agent:dev-gabedos-revert-d01d852-apm-container-tags-275edf32-jmx-zstd + image: datadog/agent-dev:275edf32-py3-jmx environment: - DD_API_KEY=${DD_API_KEY} - DD_SITE=datadoghq.com From 18afafe9d601c0ce1945d50d2038558f2f678e16 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 08:09:09 +0200 Subject: [PATCH 03/20] test again --- .../auto-inject/auto-inject_installer_manual.yml | 7 +++++-- .../weblogs/common/create_and_run_app_container.sh | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 937bf0589cc..3bf2c027961 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -10,8 +10,8 @@ remote-command: | # APMSP-3059 - export DD_INSTALLER_AGENT_VERSION=114350808 - export DD_INSTALLER_INSTALLER_VERSION=114350808 + export DD_INSTALLER_AGENT_VERSION="pipeline-114350808" + export DD_INSTALLER_INSTALLER_VERSION="pipeline-114350808" # Check if Docker is installed and ensure it's running if command -v docker >/dev/null 2>&1; then echo "Docker is installed, ensuring service is enabled and running..." @@ -47,6 +47,7 @@ if [ "${DD_env}" == "dev" ]; then # To force the installer to pull from dev repositories -- agent config is set manually to datadoghq.com + echo "RMM TEST DD_env: ${DD_env}" export DD_SITE="datad0g.com" export DD_INSTALLER_REGISTRY_URL='install.datad0g.com' #The latest_snapshot of python tracer version is 2.x we want to use 4.x. Get from repo. @@ -80,11 +81,13 @@ fi if [ -n "${DD_INSTALLER_INJECTOR_VERSION}" ]; then + echo "RMM TEST DD_INSTALLER_INJECTOR_VERSION: ${DD_INSTALLER_INJECTOR_VERSION}" export DD_INSTALLER_REGISTRY_URL_APM_INJECT_PACKAGE='installtesting.datad0g.com' export DD_INSTALLER_DEFAULT_PKG_VERSION_DATADOG_APM_INJECT="${DD_INSTALLER_INJECTOR_VERSION}" fi if [ -n "${DD_INSTALLER_AGENT_VERSION}" ]; then + echo "RMM TEST DD_INSTALLER_AGENT_VERSION: ${DD_INSTALLER_AGENT_VERSION}" export DD_INSTALLER_REGISTRY_URL_AGENT_PACKAGE='installtesting.datad0g.com' export DD_INSTALLER_DEFAULT_PKG_VERSION_DATADOG_AGENT="${DD_INSTALLER_AGENT_VERSION}" fi diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh index 378ef3d83b6..4f583ab3950 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh @@ -3,6 +3,9 @@ set -e +# Dump agent logs on exit (including failure) so health check issues are always visible +trap 'if [ -f docker-compose-agent-prod.yml ]; then echo "..:: DATADOG AGENT OUTPUT (trap) ::.." && sudo docker-compose -f docker-compose-agent-prod.yml logs datadog 2>&1 || true; fi' EXIT + # shellcheck disable=SC2035 sudo chmod -R 755 * From b771f0eb19b624a3da6dc73b66f4ba43fa75e70e Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 09:07:27 +0200 Subject: [PATCH 04/20] dump logs --- .../common/create_and_run_app_container.sh | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh index 4f583ab3950..e9c53c97588 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh @@ -3,8 +3,30 @@ set -e -# Dump agent logs on exit (including failure) so health check issues are always visible -trap 'if [ -f docker-compose-agent-prod.yml ]; then echo "..:: DATADOG AGENT OUTPUT (trap) ::.." && sudo docker-compose -f docker-compose-agent-prod.yml logs datadog 2>&1 || true; fi' EXIT +_dd_agent_diagnostics_dumped=0 + +dump_dd_agent_diagnostics() { + if [ "$_dd_agent_diagnostics_dumped" -eq 1 ]; then + return 0 + fi + if [ ! -f docker-compose-agent-prod.yml ]; then + return 0 + fi + _dd_agent_diagnostics_dumped=1 + echo "..:: DD-AGENT DIAGNOSTICS ::.." + sudo docker-compose -f docker-compose-agent-prod.yml ps 2>&1 || true + if sudo docker inspect dd-agent >/dev/null 2>&1; then + echo "..:: DD-AGENT HEALTH ::.." + sudo docker inspect dd-agent --format '{{json .State.Health}}' 2>&1 || true + echo "..:: DD-AGENT LOGS (docker logs) ::.." + sudo docker logs dd-agent 2>&1 | tail -300 || true + fi + echo "..:: DD-AGENT LOGS (docker-compose logs) ::.." + sudo docker-compose -f docker-compose-agent-prod.yml logs --no-color datadog 2>&1 || true +} + +# Dump agent diagnostics on any failure (deduplicated if already printed) +trap 'status=$?; if [ "$status" -ne 0 ]; then dump_dd_agent_diagnostics; fi; exit "$status"' EXIT # shellcheck disable=SC2035 sudo chmod -R 755 * @@ -41,9 +63,12 @@ done if [ -f docker-compose-agent-prod.yml ]; then # Agent may be installed in a different way echo "DD_API_KEY=${DD_API_KEY}" > .env - sudo -E docker-compose -f docker-compose-agent-prod.yml up -d --remove-orphans datadog --wait --wait-timeout 120 + if ! sudo -E docker-compose -f docker-compose-agent-prod.yml up -d --remove-orphans datadog --wait --wait-timeout 120; then + dump_dd_agent_diagnostics + exit 1 + fi fi -#Env variables set on the scenario definition. Write to file and load +#Env variables set on the scenario definition. Write to file and load if [ ! -f scenario_app.env ] then SCENARIO_APP_ENV="${DD_APP_ENV:-''}" @@ -53,7 +78,7 @@ then fi sudo -E docker-compose -f docker-compose.yml up -d test-app -echo "..:: RUNNING DOCKER SERVICES ::.." +echo "..:: RUNNING DOCKER SERVICES ::.." sudo docker-compose ps if [ -f docker-compose-agent-prod.yml ]; then echo "..:: DATADOG AGENT OUTPUT ::.." From b4fc3183f8f4bd556b8cdc59c000df22e1f31893 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 09:54:37 +0200 Subject: [PATCH 05/20] variables --- .../auto-inject/auto-inject_installer_manual.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 3bf2c027961..c95cdee97fc 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -12,6 +12,13 @@ # APMSP-3059 export DD_INSTALLER_AGENT_VERSION="pipeline-114350808" export DD_INSTALLER_INSTALLER_VERSION="pipeline-114350808" + export TESTING_APT_URL="apttesting.datad0g.com" + export TESTING_APT_REPO_VERSION="pipeline-114350808-a7-x86_64 7" + export TESTING_YUM_URL="yumtesting.datad0g.com" + export TESTING_YUM_VERSION_PATH="testing/pipeline-114350808-a7/7" + + + # Check if Docker is installed and ensure it's running if command -v docker >/dev/null 2>&1; then echo "Docker is installed, ensuring service is enabled and running..." From 3ff7450ca42da35d2ebcd9e2ea2226ac10a964e1 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 10:12:58 +0200 Subject: [PATCH 06/20] ecr registry --- .../provisions/auto-inject/docker/docker-compose-agent-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index 5d4f269bcea..8917edae863 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -2,7 +2,7 @@ services: datadog: container_name: dd-agent # Pin to 7.78.4 agent release. APMSP-3059 - image: datadog/agent-dev:275edf32-py3-jmx + image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:275edf32-py3-jmx environment: - DD_API_KEY=${DD_API_KEY} - DD_SITE=datadoghq.com From 8b667dfd06cb733a61d232cc57bfacfa97d6ad3d Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 11:25:44 +0200 Subject: [PATCH 07/20] test again --- .../auto-inject/auto-inject_installer_manual.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index c95cdee97fc..14b6604c959 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -10,12 +10,12 @@ remote-command: | # APMSP-3059 - export DD_INSTALLER_AGENT_VERSION="pipeline-114350808" - export DD_INSTALLER_INSTALLER_VERSION="pipeline-114350808" + export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" + export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" export TESTING_APT_URL="apttesting.datad0g.com" - export TESTING_APT_REPO_VERSION="pipeline-114350808-a7-x86_64 7" + export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" - export TESTING_YUM_VERSION_PATH="testing/pipeline-114350808-a7/7" + export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" From 65b926f31f1eb5f90321f59c20355461cc80bc30 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 11:55:07 +0200 Subject: [PATCH 08/20] diagnosis --- .../common/create_and_run_app_container.sh | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh index e9c53c97588..6bdff32f0b2 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh @@ -1,8 +1,16 @@ #!/bin/bash # shellcheck disable=SC2015 +# Provision runs "sh create_and_run_app_container.sh" — re-exec with bash for traps/functions. +if [ -z "${BASH_VERSION:-}" ]; then + exec /bin/bash "$0" "$@" +fi + set -e +readonly DIAGNOSTICS_LOG="${HOME}/dd-agent-diagnostics.log" +readonly SCRIPT_MARKER="create_and_run_app_container.sh diagnostics-v3" + _dd_agent_diagnostics_dumped=0 dump_dd_agent_diagnostics() { @@ -13,21 +21,37 @@ dump_dd_agent_diagnostics() { return 0 fi _dd_agent_diagnostics_dumped=1 - echo "..:: DD-AGENT DIAGNOSTICS ::.." - sudo docker-compose -f docker-compose-agent-prod.yml ps 2>&1 || true - if sudo docker inspect dd-agent >/dev/null 2>&1; then - echo "..:: DD-AGENT HEALTH ::.." - sudo docker inspect dd-agent --format '{{json .State.Health}}' 2>&1 || true - echo "..:: DD-AGENT LOGS (docker logs) ::.." - sudo docker logs dd-agent 2>&1 | tail -300 || true + + { + echo "..:: DD-AGENT DIAGNOSTICS (${SCRIPT_MARKER}) ::.." + date -u '+%Y-%m-%dT%H:%M:%SZ' + sudo docker-compose -f docker-compose-agent-prod.yml ps 2>&1 || true + if sudo docker inspect dd-agent >/dev/null 2>&1; then + echo "..:: DD-AGENT HEALTH ::.." + sudo docker inspect dd-agent --format '{{json .State.Health}}' 2>&1 || true + echo "..:: DD-AGENT LOGS (docker logs) ::.." + sudo docker logs dd-agent 2>&1 | tail -300 || true + else + echo "..:: dd-agent container not found ::.." + sudo docker ps -a 2>&1 || true + fi + echo "..:: DD-AGENT LOGS (docker-compose logs) ::.." + sudo docker-compose -f docker-compose-agent-prod.yml logs --no-color datadog 2>&1 || true + } 2>&1 | tee -a "${DIAGNOSTICS_LOG}" + + sudo mkdir -p /var/log/datadog_weblog 2>/dev/null || true + if [ -d /var/log/datadog_weblog ]; then + sudo cp "${DIAGNOSTICS_LOG}" /var/log/datadog_weblog/dd-agent-diagnostics.log 2>/dev/null || true + sudo chmod 644 /var/log/datadog_weblog/dd-agent-diagnostics.log 2>/dev/null || true fi - echo "..:: DD-AGENT LOGS (docker-compose logs) ::.." - sudo docker-compose -f docker-compose-agent-prod.yml logs --no-color datadog 2>&1 || true + sync 2>/dev/null || true } # Dump agent diagnostics on any failure (deduplicated if already printed) trap 'status=$?; if [ "$status" -ne 0 ]; then dump_dd_agent_diagnostics; fi; exit "$status"' EXIT +echo "..:: ${SCRIPT_MARKER} ::.." + # shellcheck disable=SC2035 sudo chmod -R 755 * @@ -64,7 +88,9 @@ if [ -f docker-compose-agent-prod.yml ]; then # Agent may be installed in a different way echo "DD_API_KEY=${DD_API_KEY}" > .env if ! sudo -E docker-compose -f docker-compose-agent-prod.yml up -d --remove-orphans datadog --wait --wait-timeout 120; then + echo "..:: COMPOSE_WAIT_FAILED (dd-agent unhealthy or timeout) ::.." | tee -a "${DIAGNOSTICS_LOG}" >&2 dump_dd_agent_diagnostics + echo "..:: Diagnostics written to ${DIAGNOSTICS_LOG} ::.." >&2 exit 1 fi fi From a9c40014b9bbba1e2305a75acd54133fed1cf22e Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 12:01:28 +0200 Subject: [PATCH 09/20] test again --- .../provisions/auto-inject/auto-inject_installer_manual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 14b6604c959..a1d20ae0a6b 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -13,9 +13,9 @@ export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" export TESTING_APT_URL="apttesting.datad0g.com" - export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" + #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" - export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" + #export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" From c5798d8f2b9b161bb6b9c7b19255a800cc059dcb Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 12:11:52 +0200 Subject: [PATCH 10/20] retry --- utils/_context/_scenarios/auto_injection.py | 7 +- .../common/create_and_run_app_container.sh | 4 + utils/onboarding/debug_vm.py | 130 +++++++++++------- utils/virtual_machine/aws_provider.py | 44 ++++-- utils/virtual_machine/virtual_machines.py | 6 + 5 files changed, 136 insertions(+), 55 deletions(-) diff --git a/utils/_context/_scenarios/auto_injection.py b/utils/_context/_scenarios/auto_injection.py index 874432e8cda..aae5352bb8c 100644 --- a/utils/_context/_scenarios/auto_injection.py +++ b/utils/_context/_scenarios/auto_injection.py @@ -158,7 +158,12 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None def close_targets(self): if self.is_main_worker: - # Extract logs from the VM before destroy + # Extract logs from the VM before destroy (including after provision failures) + if self.virtual_machine.provision_install_error is not None: + logger.stdout( + "Provision failed — downloading VM logs via SSH/SFTP " + f"(see {self.host_log_folder}/var/log/datadog_weblog/dd-agent-diagnostics.log)" + ) download_vm_logs( vm=self.virtual_machine, remote_folder_paths=["/var/log/datadog", "/var/log/datadog_weblog", "/tmp/datadog/java"], # noqa: S108 diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh index 6bdff32f0b2..6a786368e62 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh @@ -52,6 +52,10 @@ trap 'status=$?; if [ "$status" -ne 0 ]; then dump_dd_agent_diagnostics; fi; exi echo "..:: ${SCRIPT_MARKER} ::.." +# Writable by log download even when provision fails before vm_logs step +sudo mkdir -p /var/log/datadog_weblog 2>/dev/null || true +sudo chmod 777 /var/log/datadog_weblog 2>/dev/null || true + # shellcheck disable=SC2035 sudo chmod -R 755 * diff --git a/utils/onboarding/debug_vm.py b/utils/onboarding/debug_vm.py index 8a4508063c1..6d208d229fe 100644 --- a/utils/onboarding/debug_vm.py +++ b/utils/onboarding/debug_vm.py @@ -4,98 +4,136 @@ from utils._logger import logger from utils.virtual_machine.virtual_machines import _VirtualMachine - -def download_vm_logs(vm: _VirtualMachine, remote_folder_paths: list[str], local_base_logs_folder: str): - """Using SSH/SFTP connects to VM and downloads one or more folders from the remote machine - - Args: - vm: Virtual machine object - remote_folder_paths: Single path (str) or list of paths (list[str]) to download - local_base_logs_folder: Base folder where logs will be stored locally - +# Collect dd-agent docker diagnostics into /var/log/datadog_weblog (runs from VM user home). +_COLLECT_DD_AGENT_DIAGNOSTICS_CMD = r"""bash -lc ' +sudo mkdir -p /var/log/datadog_weblog && sudo chmod 777 /var/log/datadog_weblog; +cd ~; +if [ -f "$HOME/dd-agent-diagnostics.log" ]; then + sudo cp "$HOME/dd-agent-diagnostics.log" /var/log/datadog_weblog/dd-agent-diagnostics.log 2>/dev/null || true; +fi; +if [ -f docker-compose-agent-prod.yml ]; then +{ +echo "..:: DD-AGENT DIAGNOSTICS (download_vm_logs) ::.."; +date -u "+%Y-%m-%dT%H:%M:%SZ"; +sudo docker-compose -f docker-compose-agent-prod.yml ps 2>&1 || true; +if sudo docker inspect dd-agent >/dev/null 2>&1; then +echo "..:: DD-AGENT HEALTH ::.."; +sudo docker inspect dd-agent --format "{{json .State.Health}}" 2>&1 || true; +echo "..:: DD-AGENT LOGS (docker logs) ::.."; +sudo docker logs dd-agent 2>&1 | tail -300 || true; +else echo "..:: dd-agent container not found ::.."; sudo docker ps -a 2>&1 || true; +fi; +echo "..:: DD-AGENT LOGS (docker-compose logs) ::.."; +sudo docker-compose -f docker-compose-agent-prod.yml logs --no-color datadog 2>&1 || true; +} | sudo tee /var/log/datadog_weblog/dd-agent-diagnostics.log >/dev/null; +fi'""" + + +def download_vm_logs(vm: _VirtualMachine, remote_folder_paths: list[str], local_base_logs_folder: str) -> bool: + """Connect over SSH/SFTP and download folders from the remote machine. + + Works even when provisioning failed (uses get_ssh_connection_for_log_download). + + Returns True if at least one folder was downloaded successfully. """ - # Handle both single path and list of paths for backward compatibility if isinstance(remote_folder_paths, str): remote_folder_paths = [remote_folder_paths] + if not vm.ssh_config.hostname: + logger.warning( + "Skipping VM log download for %s: no IP/hostname (VM may not have been created)", + vm.name, + ) + return False + + downloaded_any = False try: - logger.info(f"Downloading folders from machine {vm.get_ip()}") - logger.info(f"Remote folders: {remote_folder_paths}") + logger.info( + "Downloading folders from machine %s (%s) provision_error=%s", + vm.name, + vm.ssh_config.hostname, + vm.provision_install_error is not None, + ) + logger.info("Remote folders: %s", remote_folder_paths) - # Use the VM's get_ssh_connection method instead of creating our own - c = vm.get_ssh_connection() - logger.info(f"Connected [{vm.get_ip()}]") + connection = vm.get_ssh_connection_for_log_download() + logger.info("Connected [%s]", vm.ssh_config.hostname) - # Execute remote commands to collect logs prior to download commands_to_run = [ "sudo mkdir -p /var/log/datadog_weblog || true", - # Docker and systemd related logs (mirrors utils/build/virtual_machine/provisions/auto-inject/auto-inject-vm_logs.yml) - "sudo docker-compose ps > /var/log/datadog_weblog/docker_proccess.log 2>&1 || true", - "sudo docker-compose logs > /var/log/datadog_weblog/docker_logs.log 2>&1 || true", + "sudo chmod 777 /var/log/datadog_weblog || true", + _COLLECT_DD_AGENT_DIAGNOSTICS_CMD, + # Docker and systemd related logs (mirrors auto-inject-vm_logs.yml) + "bash -lc 'cd ~ && sudo docker-compose ps > /var/log/datadog_weblog/docker_proccess.log 2>&1 || true'", + "bash -lc 'cd ~ && sudo docker-compose logs > /var/log/datadog_weblog/docker_logs.log 2>&1 || true'", "sudo journalctl -xeu docker > /var/log/datadog_weblog/journalctl_docker.log 2>&1 || true", - # Copy Datadog Agent configuration files "sudo cp /etc/datadog-agent/application_monitoring.yaml /var/log/datadog_weblog/application_monitoring.yaml 2>&1 || true", - # Additional logs requested "sudo cat /var/log/cloud-init.log > /var/log/datadog_weblog/cloud-init.log 2>&1 || true", "sudo cat /var/log/syslog > /var/log/datadog_weblog/syslog.log 2>&1 || true", "sudo dmesg > /var/log/datadog_weblog/dmesg.log 2>&1 || true", "sudo systemctl list-dependencies docker.service > /var/log/datadog_weblog/docker_list_dependencies.log 2>&1 || true", "sudo systemctl list-timers --all > /var/log/datadog_weblog/system.timers.log 2>&1 || true", "sudo crontab -l > /var/log/datadog_weblog/crontab.log 2>&1 || true", - "sudo cat /var/log/apt/history.log > /var/log/datadog_weblog/apt.log 2>&1 || true", - "sudo cat /var/log/yum.log > /var/log/datadog_weblog/yum.log 2>&1 || true", + "sudo cat /var/log/apt/history.log > /var/log/datadog_weblog/apt.log 2>&1 || true", + "sudo cat /var/log/yum.log > /var/log/datadog_weblog/yum.log 2>&1 || true", ] for cmd in commands_to_run: try: - logger.info(f"Executing remote command: {cmd}") - _stdin, stdout, _stderr = c.exec_command(cmd) + logger.info("Executing remote command: %s", cmd) + _stdin, stdout, _stderr = connection.exec_command(cmd) exit_status = stdout.channel.recv_exit_status() - logger.info(f"Remote command exit status: {exit_status}") + logger.info("Remote command exit status: %s", exit_status) except Exception as exec_err: - logger.warning(f"Failed executing command on {vm.get_ip()}: {cmd}") + logger.warning("Failed executing command on %s: %s", vm.ssh_config.hostname, cmd) logger.exception(exec_err) - # Create SFTP client - sftp = c.open_sftp() + sftp = connection.open_sftp() - # Download each folder for remote_folder_path in remote_folder_paths: local_folder_path = f"{local_base_logs_folder}/{remote_folder_path}" - logger.info(f"Downloading: {remote_folder_path} -> {local_folder_path}") + logger.info("Downloading: %s -> %s", remote_folder_path, local_folder_path) - # Create local directory if it doesn't exist local_path = Path(local_folder_path) local_path.mkdir(parents=True, exist_ok=True) - # Download the folder recursively - _download_folder_recursive(sftp, remote_folder_path, local_folder_path) + if _download_folder_recursive(sftp, remote_folder_path, local_folder_path): + downloaded_any = True sftp.close() - c.close() - logger.info(f"Successfully downloaded all folders from {vm.get_ip()}") + connection.close() + if downloaded_any: + logger.info( + "Successfully downloaded VM logs from %s into %s", vm.ssh_config.hostname, local_base_logs_folder + ) + else: + logger.warning("No files downloaded from %s", vm.ssh_config.hostname) except Exception: - logger.error("Cannot download folders from remote machine") + logger.exception("Cannot download folders from remote machine %s", vm.name) + + return downloaded_any -def _download_folder_recursive(sftp: SFTPClient, remote_dir: str, local_dir: str): - """Recursively download a folder using SFTP""" +def _download_folder_recursive(sftp: SFTPClient, remote_dir: str, local_dir: str) -> bool: + """Recursively download a folder using SFTP. Returns True if at least one file was downloaded.""" + downloaded_any = False try: - # List contents of remote directory for item in sftp.listdir_attr(remote_dir): remote_path = f"{remote_dir}/{item.filename}" local_path = Path(local_dir) / item.filename if stat.S_ISDIR(item.st_mode): - # Create local directory and recursively download local_path.mkdir(exist_ok=True) - logger.info(f"Created directory: {local_path}") - _download_folder_recursive(sftp, remote_path, str(local_path)) + logger.info("Created directory: %s", local_path) + if _download_folder_recursive(sftp, remote_path, str(local_path)): + downloaded_any = True else: - # Download file - logger.info(f"Downloading file: {remote_path} -> {local_path}") + logger.info("Downloading file: %s -> %s", remote_path, local_path) sftp.get(remote_path, str(local_path)) + downloaded_any = True except Exception: - logger.error(f"Error downloading from {remote_dir}") + logger.exception("Error downloading from %s", remote_dir) + + return downloaded_any diff --git a/utils/virtual_machine/aws_provider.py b/utils/virtual_machine/aws_provider.py index 771a0651260..373181885df 100644 --- a/utils/virtual_machine/aws_provider.py +++ b/utils/virtual_machine/aws_provider.py @@ -22,11 +22,14 @@ from utils import context from utils.virtual_machine.vm_logger import vm_logger +from utils.onboarding.debug_vm import download_vm_logs from utils.virtual_machine.virtual_machine_provider import VmProvider, Commander from utils.virtual_machine.virtual_machines import _VirtualMachine import pytest +_VM_LOG_REMOTE_PATHS = ["/var/log/datadog", "/var/log/datadog_weblog", "/tmp/datadog/java"] # noqa: S108 + class AWSPulumiProvider(VmProvider): def __init__(self): @@ -133,12 +136,29 @@ def _handle_provision_error(self, exception: pulumi.automation.errors.CommandErr pytest.exit(f"Known infraestructure exception:: {known_message}", returncode=3) # If the exception is not known, we will store it in the vm object and error event to dd self.vm.provision_install_error = exception + self._download_vm_logs_after_provision_failure() self.datadog_event_sender.sendEventToDatadog( f"[E2E] Stack {self.stack_name} : error on Pulumi stack up", repr(exception), ["operation:up", "result:fail", f"stack:{self.stack_name}"], ) + def _download_vm_logs_after_provision_failure(self) -> None: + """Pull /var/log from the VM while it is still up (provision may have failed mid-way).""" + if not self.vm.ssh_config.hostname: + logger.warning("Cannot download VM logs: VM has no IP yet") + return + log_folder = context.scenario.host_log_folder + logger.stdout( + f"Downloading VM logs after provision failure into {log_folder} " + "(including var/log/datadog_weblog/dd-agent-diagnostics.log if present)" + ) + download_vm_logs( + vm=self.vm, + remote_folder_paths=_VM_LOG_REMOTE_PATHS, + local_base_logs_folder=log_folder, + ) + def _start_vm(self, vm: _VirtualMachine): ec2_user_data = None if vm.os_type == "windows": @@ -422,18 +442,26 @@ def remote_command( lambda outputlog: vm_logger(context.scenario.host_log_folder, logger_name).info(outputlog) ) else: - # If there isn't logger name specified, we will use the host/ip name to store all the logs of the - # same remote machine in the same log file - # If there is a failure (exit != 0) this trace will not be stored in the log file (why?) header = "\n *****************************************************************" header2 = "\n -----------------------------------------------------------------" + + def _log_remote_command_output(args: list[object]) -> None: + vm_name = str(args[0]) + step_id = str(args[1]) + command = str(args[2]) + stdout = str(args[3] or "") + stderr = str(args[4] or "") + failed = "Process exited with status" in stderr or "error:" in stderr.lower() + status = "❌ FAILED" if failed else "✅ SUCCESS" + vm_logger(context.scenario.host_log_folder, vm_name).info( + f"{header} \n 🚀 Provision step: {step_id} 🚀 \n Status: {status} \n {header} \n" + f" {command} \n\n {header2} \n 📤 Provision step output ({step_id}) 📤 \n {header2} \n" + f" {stdout} \n\n {header2} \n 🚨 error output ({step_id}) 🚨 \n {header2} \n {stderr}" + ) + Output.all( vm.name, installation_id, remote_command, cmd_exec_install.stdout, cmd_exec_install.stderr - ).apply( - lambda args: vm_logger(context.scenario.host_log_folder, args[0]).info( - f"{header} \n 🚀 Provision step: {args[1]} 🚀 \n Status: ✅ SUCCESS \n {header} \n {args[2]} \n\n {header2} \n 📤 Provision step output ({args[1]}) 📤 \n {header2} \n {args[3]} \n\n {header2} \n 🚨 error output ({args[1]}) 🚨 \n {header2} \n {args[4]}" - ) - ) + ).apply(_log_remote_command_output) if output_callback: Output.all(vm, cmd_exec_install.stdout).apply(output_callback) diff --git a/utils/virtual_machine/virtual_machines.py b/utils/virtual_machine/virtual_machines.py index a51bcda0797..38327e5e3df 100644 --- a/utils/virtual_machine/virtual_machines.py +++ b/utils/virtual_machine/virtual_machines.py @@ -141,6 +141,12 @@ def get_ssh_connection(self): self._check_provsion_install_error() return self.ssh_config.get_ssh_connection() + def get_ssh_connection_for_log_download(self): + """SSH for log extraction after a failed provision (skips provision_install_error guard).""" + if not self.ssh_config.hostname: + raise Exception("IP not found - cannot download logs from VM") + return self.ssh_config.get_ssh_connection() + def get_ip(self): self._check_provsion_install_error() if not self.ssh_config.hostname: From c1628ee2ec8766e8931b7709005fb3ed4a67c470 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 15:25:39 +0200 Subject: [PATCH 11/20] test --- .../provisions/auto-inject/docker/docker-compose-agent-prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index 8917edae863..edee7b1e9de 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -5,6 +5,8 @@ services: image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:275edf32-py3-jmx environment: - DD_API_KEY=${DD_API_KEY} + # EC2 Docker: explicit hostname required (Agent >=7.40, APMSP-3059) + - DD_HOSTNAME=system-tests-ssi-agent - DD_SITE=datadoghq.com - DD_APM_ENABLED=true - DD_LOG_LEVEL=TRACE From e95295dcb5031fdade835f37b2ff88bae7aeb574 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 15:55:11 +0200 Subject: [PATCH 12/20] test new image tag --- .../auto-inject/docker/docker-compose-agent-prod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index edee7b1e9de..cb9d707b112 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -2,7 +2,8 @@ services: datadog: container_name: dd-agent # Pin to 7.78.4 agent release. APMSP-3059 - image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:275edf32-py3-jmx + #image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:275edf32-py3-jmx + image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:f58ff1f1-py3-jmx environment: - DD_API_KEY=${DD_API_KEY} # EC2 Docker: explicit hostname required (Agent >=7.40, APMSP-3059) From ea68ca0d8648b58697ffc477856b68bce85e2cc0 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 16:18:37 +0200 Subject: [PATCH 13/20] without hostname --- .../provisions/auto-inject/docker/docker-compose-agent-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index cb9d707b112..ed26b2362d7 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -7,7 +7,7 @@ services: environment: - DD_API_KEY=${DD_API_KEY} # EC2 Docker: explicit hostname required (Agent >=7.40, APMSP-3059) - - DD_HOSTNAME=system-tests-ssi-agent + #- DD_HOSTNAME=system-tests-ssi-agent - DD_SITE=datadoghq.com - DD_APM_ENABLED=true - DD_LOG_LEVEL=TRACE From 0bbe9d0467f2c42087679924b71e465d793ece04 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 16:21:32 +0200 Subject: [PATCH 14/20] testing keys for simple_installer scenario --- .../provisions/auto-inject/auto-inject_installer_manual.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index a1d20ae0a6b..8ad99229034 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -16,7 +16,7 @@ #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" #export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" - + export TESTING_KEYS_URL='apttesting.datad0g.com/test-keys' # Check if Docker is installed and ensure it's running From 5bbcb709d7aa4a0ea063490eda28038561aff372 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 16:45:01 +0200 Subject: [PATCH 15/20] remove the installer version --- .../provisions/auto-inject/auto-inject_installer_manual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 8ad99229034..d7138f805b3 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -11,12 +11,12 @@ remote-command: | # APMSP-3059 export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" - export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" + #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" export TESTING_APT_URL="apttesting.datad0g.com" #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" #export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" - export TESTING_KEYS_URL='apttesting.datad0g.com/test-keys' + export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" # Check if Docker is installed and ensure it's running From d9bd1205a7aabd74b99ca87da21cd719fadc0b45 Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 17:23:20 +0200 Subject: [PATCH 16/20] test agaiin --- .../auto-inject/auto-inject_installer_manual.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index d7138f805b3..8ddeaf5e1d0 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -10,14 +10,21 @@ remote-command: | # APMSP-3059 - export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" + #export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" - export TESTING_APT_URL="apttesting.datad0g.com" + #export TESTING_APT_URL="apttesting.datad0g.com" #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" - export TESTING_YUM_URL="yumtesting.datad0g.com" + #export TESTING_YUM_URL="yumtesting.datad0g.com" #export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" - export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" + #export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" + export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" + export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" + export TESTING_APT_URL="apttesting.datad0g.com" + export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" + export TESTING_YUM_URL="yumtesting.datad0g.com" + export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" + export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" # Check if Docker is installed and ensure it's running if command -v docker >/dev/null 2>&1; then From 9175727ae20f3c2a9ff84c991ce65cf0a13a151d Mon Sep 17 00:00:00 2001 From: robertomonteromiguel Date: Fri, 22 May 2026 18:10:10 +0200 Subject: [PATCH 17/20] check again --- .../provisions/auto-inject/auto-inject_installer_manual.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 8ddeaf5e1d0..e32e1ed6295 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -12,6 +12,7 @@ # APMSP-3059 #export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" + #export TESTING_APT_URL="apttesting.datad0g.com" #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" #export TESTING_YUM_URL="yumtesting.datad0g.com" @@ -20,8 +21,8 @@ export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" - export TESTING_APT_URL="apttesting.datad0g.com" - export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" + export TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-114327295-a7" + export TESTING_APT_REPO_VERSION="stable-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" From 412e455076bf67d9b8c6283f24f3835c4b4b5fbb Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Fri, 22 May 2026 13:22:43 -0400 Subject: [PATCH 18/20] try newer fix --- .../auto-inject/auto-inject_installer_manual.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index e32e1ed6295..52e7ea8fa7d 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -10,21 +10,21 @@ remote-command: | # APMSP-3059 - #export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" - #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" + #export DD_INSTALLER_AGENT_VERSION="pipeline-114793458" + #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114793458" #export TESTING_APT_URL="apttesting.datad0g.com" - #export TESTING_APT_REPO_VERSION="pipeline-114327295-a7-x86_64 7" + #export TESTING_APT_REPO_VERSION="pipeline-114793458-a7-x86_64 7" #export TESTING_YUM_URL="yumtesting.datad0g.com" - #export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" + #export TESTING_YUM_VERSION_PATH="testing/pipeline-114793458-a7/7" #export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" - export DD_INSTALLER_AGENT_VERSION="pipeline-114327295" - export DD_INSTALLER_INSTALLER_VERSION="pipeline-114327295" - export TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-114327295-a7" + export DD_INSTALLER_AGENT_VERSION="pipeline-114793458" + export DD_INSTALLER_INSTALLER_VERSION="pipeline-114793458" + export TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-114793458-a7" export TESTING_APT_REPO_VERSION="stable-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" - export TESTING_YUM_VERSION_PATH="testing/pipeline-114327295-a7/7" + export TESTING_YUM_VERSION_PATH="testing/pipeline-114793458-a7/7" export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" # Check if Docker is installed and ensure it's running From ab8dd478cdb2dadd393acabd23c228e0910d539e Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Fri, 22 May 2026 15:53:13 -0400 Subject: [PATCH 19/20] try new commit with BOTH fixes --- .../provisions/auto-inject/docker/docker-compose-agent-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index ed26b2362d7..a8c0ed82e84 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -3,7 +3,7 @@ services: container_name: dd-agent # Pin to 7.78.4 agent release. APMSP-3059 #image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:275edf32-py3-jmx - image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:f58ff1f1-py3-jmx + image: 669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub/datadog/agent-dev:623e4f63-py3-jmx environment: - DD_API_KEY=${DD_API_KEY} # EC2 Docker: explicit hostname required (Agent >=7.40, APMSP-3059) From 3abeca4929d9a84065d38fa9a3cac3d8830d7e9c Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Fri, 22 May 2026 16:05:04 -0400 Subject: [PATCH 20/20] try both together but also for the agent on pure host install --- .../auto-inject/auto-inject_installer_manual.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 52e7ea8fa7d..61439ac0570 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -10,21 +10,21 @@ remote-command: | # APMSP-3059 - #export DD_INSTALLER_AGENT_VERSION="pipeline-114793458" - #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114793458" + #export DD_INSTALLER_AGENT_VERSION="pipeline-114830198" + #export DD_INSTALLER_INSTALLER_VERSION="pipeline-114830198" #export TESTING_APT_URL="apttesting.datad0g.com" - #export TESTING_APT_REPO_VERSION="pipeline-114793458-a7-x86_64 7" + #export TESTING_APT_REPO_VERSION="pipeline-114830198-a7-x86_64 7" #export TESTING_YUM_URL="yumtesting.datad0g.com" - #export TESTING_YUM_VERSION_PATH="testing/pipeline-114793458-a7/7" + #export TESTING_YUM_VERSION_PATH="testing/pipeline-114830198-a7/7" #export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" - export DD_INSTALLER_AGENT_VERSION="pipeline-114793458" - export DD_INSTALLER_INSTALLER_VERSION="pipeline-114793458" - export TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-114793458-a7" + export DD_INSTALLER_AGENT_VERSION="pipeline-114830198" + export DD_INSTALLER_INSTALLER_VERSION="pipeline-114830198" + export TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-114830198-a7" export TESTING_APT_REPO_VERSION="stable-x86_64 7" export TESTING_YUM_URL="yumtesting.datad0g.com" - export TESTING_YUM_VERSION_PATH="testing/pipeline-114793458-a7/7" + export TESTING_YUM_VERSION_PATH="testing/pipeline-114830198-a7/7" export TESTING_KEYS_URL="apttesting.datad0g.com/test-keys" # Check if Docker is installed and ensure it's running