Skip to content

Commit 151082e

Browse files
JohnMalmbergJohnMalmberg
committed
SRE-3842 build: NOT FOR LANDING
This is a test to induce some errors to force the testing of some error handling code to validate that it is working and collect some diagnostic infomation. Signed-off-by: John E. Malmberg <john.malmberg@hpe.com>
1 parent ee9924b commit 151082e

4 files changed

Lines changed: 98 additions & 15 deletions

File tree

‎Jenkinsfile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -453,7 +453,8 @@ pipeline {
453453
stage('Python Bandit check') {
454454
when {
455455
beforeAgent true
456-
expression { !skipStage() }
456+
// TESTING: disable for forced-failure diagnostics - REMOVE AFTER TEST
457+
expression { false }
457458
}
458459
agent {
459460
dockerfile {
@@ -493,7 +494,8 @@ pipeline {
493494
stage('Build on EL 8') {
494495
when {
495496
beforeAgent true
496-
expression { !skip_build_stage('el8') }
497+
// TESTING: disable for forced-failure diagnostics - REMOVE AFTER TEST
498+
expression { false }
497499
}
498500
agent {
499501
dockerfile {
@@ -601,7 +603,8 @@ pipeline {
601603
stage('Build on Leap 15') {
602604
when {
603605
beforeAgent true
604-
expression { !skip_build_stage('leap15') }
606+
// TESTING: disable for forced-failure diagnostics - REMOVE AFTER TEST
607+
expression { false }
605608
}
606609
agent {
607610
dockerfile {

‎ci/provisioning/post_provision_config.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,17 @@ fi
8181

8282
: "${DAOS_FAILURE_STEP:=startup}"
8383

84+
# ---------------------------------------------------------------------------
85+
# TESTING: Speed up retries for mail diagnostics debugging - REMOVE AFTER TEST
86+
# ---------------------------------------------------------------------------
87+
export RETRY_COUNT=1
88+
export RETRY_DELAY_SECONDS=1
89+
export DAOS_STACK_RETRY_COUNT=1
90+
export DAOS_STACK_RETRY_DELAY_SECONDS=1
91+
# ---------------------------------------------------------------------------
92+
# END TESTING BLOCK - DELETE ABOVE 4 LINES AFTER COLLECTING DEBUG LOGS
93+
# ---------------------------------------------------------------------------
94+
8495
DAOS_FAILURE_STEP="copy_ci_keys"
8596
retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c ci_key* --dest=/tmp/
8697

‎ci/provisioning/post_provision_config_common_functions.sh‎

Lines changed: 71 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,39 @@ retry_dnf() {
135135
send_mail() {
136136
local subject="$1"
137137
local message="${2:-}"
138-
139-
local recipients
140-
if mail --help | grep s-nail; then
141-
recipients="${OPERATIONS_EMAIL//, }"
142-
else
143-
recipients="${OPERATIONS_EMAIL}"
144-
fi
138+
# debug start
139+
local artifact_dir="/var/tmp/artifacts"
140+
local log_file="${artifact_dir}/post_provision_mail_debug.log"
141+
local ts
142+
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
143+
mkdir -p "$artifact_dir"
144+
# debug end
145+
146+
# debug start - force single recipient during diagnostics testing
147+
local recipients="john.malmberg@hpe.com"
148+
# debug end
149+
150+
local mail_cmd
151+
printf -v mail_cmd 'mail -s %q -r %q %q' "$subject" "$DAOS_DEVOPS_EMAIL" "$recipients"
152+
153+
# debug start
154+
{
155+
echo "[$ts] send_mail invoked"
156+
echo "subject=$subject"
157+
echo "from=$DAOS_DEVOPS_EMAIL"
158+
echo "to=$recipients"
159+
echo "mail_cmd=$mail_cmd"
160+
echo "build=$BUILD_URL"
161+
echo "stage=$STAGE_NAME"
162+
echo "host=$HOSTNAME"
163+
echo "env snapshot (allowlist):"
164+
printenv | grep -E '^(BUILD_URL|STAGE_NAME|HOSTNAME|OPERATIONS_EMAIL|DAOS_DEVOPS_EMAIL|JENKINS_URL|EMAIL_DOMAIN|DAOS_FAILURE_STEP|NODESTRING)=' | sort
165+
echo "env snapshot (full - debug only):"
166+
printenv | sort
167+
echo "postfix main.cf (postconf -n):"
168+
postconf -n 2>&1 || true
169+
} >> "$log_file" 2>&1
170+
# debug end
145171

146172
set +x
147173
{
@@ -150,7 +176,32 @@ send_mail() {
150176
echo "Host: $HOSTNAME"
151177
echo ""
152178
echo -e "$message"
153-
} 2>&1 | mail -s "$subject" -r "$DAOS_DEVOPS_EMAIL" "$recipients"
179+
# debug start - mail output changed to log to file
180+
# Original (remove >> "$log_file" 2>&1 when reverting):
181+
# } 2>&1 | mail -s "$subject" -r "$DAOS_DEVOPS_EMAIL" "$recipients"
182+
} 2>&1 | mail -s "$subject" -r "$DAOS_DEVOPS_EMAIL" "$recipients" \
183+
>> "$log_file" 2>&1
184+
# debug end
185+
186+
# debug start
187+
{
188+
echo "[$ts] postqueue -p"
189+
postqueue -p
190+
} >> "$log_file" 2>&1 || true
191+
192+
{
193+
echo "[$ts] postfix config snapshot"
194+
if command -v postconf >/dev/null 2>&1; then
195+
postconf -nh relayhost
196+
postconf -nh smtp_host_lookup
197+
postconf -nh smtp_tls_security_level
198+
postconf -nh myhostname
199+
postconf -nh myorigin
200+
else
201+
echo "postconf not found"
202+
fi
203+
} >> "$log_file" 2>&1 || true
204+
# debug end
154205
set -x
155206
}
156207

@@ -160,8 +211,10 @@ monitor_cmd() {
160211

161212
local duration=0
162213
local start="$SECONDS"
163-
if ! time "$@"; then
164-
return "${PIPESTATUS[0]}"
214+
if time "$@"; then
215+
:
216+
else
217+
return "$?"
165218
fi
166219
((duration = SECONDS - start))
167220
if [ "$duration" -gt "$threshold" ]; then
@@ -185,17 +238,20 @@ retry_cmd() {
185238
"Command: $*\nAttempts: $attempt\nStatus: $rc"
186239
fi
187240
return 0
241+
else
242+
rc=$?
188243
fi
189244
# Command failed, retry
190-
rc=${PIPESTATUS[0]}
191245
(( attempt++ )) || true
192246
if [ "$attempt" -gt 0 ]; then
193247
sleep "${RETRY_DELAY_SECONDS:-$DAOS_STACK_RETRY_DELAY_SECONDS}"
194248
fi
195249
done
196250
if [ "$rc" -ne 0 ]; then
251+
echo "DEBUG: retry_cmd invoking send_mail (attempts=$attempt rc=$rc stage=$STAGE_NAME)"
197252
send_mail "Command retry failed in $STAGE_NAME after $attempt attempts" \
198253
"Command: $*\nAttempts: $attempt\nStatus: $rc"
254+
echo "DEBUG: retry_cmd returned from send_mail"
199255
echo "Command retry failed in $STAGE_NAME after $attempt attempts"
200256
echo "Command: $*"
201257
echo "Attempts: $attempt"
@@ -218,8 +274,9 @@ timeout_cmd() {
218274
"Command: $*\nAttempts: $attempt\nStatus: $rc"
219275
fi
220276
return 0
277+
else
278+
rc=$?
221279
fi
222-
rc=${PIPESTATUS[0]}
223280
if [ "$rc" = "124" ]; then
224281
# Command timed out, try again
225282
(( attempt++ )) || true
@@ -229,8 +286,10 @@ timeout_cmd() {
229286
break
230287
done
231288
if [ "$rc" -ne 0 ]; then
289+
echo "DEBUG: timeout_cmd invoking send_mail (attempts=$attempt rc=$rc stage=$STAGE_NAME)"
232290
send_mail "Command timeout failed in $STAGE_NAME after $attempt attempts" \
233291
"Command: $*\nAttempts: $attempt\nStatus: $rc"
292+
echo "DEBUG: timeout_cmd returned from send_mail"
234293
echo "Command timeout failed in $STAGE_NAME after $attempt attempts"
235294
echo "Command: $*"
236295
echo "Attempts: $attempt"

‎ci/provisioning/post_provision_config_nodes.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,16 @@ if lspci | grep -i nvme; then
7979
fi
8080

8181

82+
# ---------------------------------------------------------------------------
83+
# TESTING: Mail diagnostics debugging - REMOVE AFTER TEST
84+
# ---------------------------------------------------------------------------
85+
DAOS_FAILURE_STEP="test_error_for_mail_debug"
86+
echo "TESTING: Forcing intentional failure to test mail diagnostics and artifact collection"
87+
RETRY_COUNT=1 RETRY_DELAY_SECONDS=1 DAOS_STACK_RETRY_COUNT=1 DAOS_STACK_RETRY_DELAY_SECONDS=1 retry_cmd 30 false
88+
# ---------------------------------------------------------------------------
89+
# END TESTING BLOCK - DELETE ABOVE 5 LINES AFTER COLLECTING DEBUG LOGS
90+
# ---------------------------------------------------------------------------
91+
8292
systemctl enable nfs-server.service
8393
systemctl start nfs-server.service
8494
sync

0 commit comments

Comments
 (0)