pipeline-build-validation.groovy

def run(params) {
    timestamps {
        //Capybara configuration
        def capybara_timeout = 60
        def default_timeout = 500
        env.bootstrap_timeout = 800

        env.controller_hostname = null
        GString TestEnvironmentCleanerProgram = "${WORKSPACE}/susemanager-ci/jenkins_pipelines/scripts/test_environment_cleaner/test_environment_cleaner_program/TestEnvironmentCleaner.py"

        deployed = false
        env.resultdir = "${WORKSPACE}/results"
        env.resultdirbuild = "${resultdir}/${BUILD_NUMBER}"
        GString localSumaformDirPath = "${resultdir}/sumaform/"
        // The junit plugin doesn't affect full paths
        GString junit_resultdir = "results/${BUILD_NUMBER}/results_junit"
        env.exports = "export BUILD_NUMBER=${BUILD_NUMBER}; export BUILD_VALIDATION=true; export CAPYBARA_TIMEOUT=${capybara_timeout}; export DEFAULT_TIMEOUT=${default_timeout}; export CUCUMBER_PUBLISH_QUIET=true;"

        // Declare lock resource use during node bootstrap
        mgrCreateBootstrapRepo = 'share resource to avoid running mgr create bootstrap repo in parallel'
        retailProxyConfigurationLock = 'lock proxy retail setup'
        // Variables to store none critical stage run status
        def monitoring_stage_result_fail = false
        def client_stage_result_fail = false
        def products_and_salt_migration_stage_result_fail = false
        def retail_stage_result_fail = false
        def containerization_stage_result_fail = false
        def server_container_repository = params.server_container_repository ?: null
        def proxy_container_repository = params.proxy_container_repository ?: null
        def server_container_image = params.server_container_image ?: ''
        // Parameters used for continuous pipeline
        def product_version = params.product_version ?: ''
        def base_os = params.base_os ?: ''

        env.common_params = "--outputdir ${resultdir} --tf ${params.tf_file} --gitfolder ${resultdir}/sumaform --terraform-bin ${params.bin_path}"

        if (params.deploy_parallelism) {
            env.common_params = "${env.common_params} --parallelism ${params.deploy_parallelism}"
        }
        try {
            stage('Clone terracumber, susemanager-ci') {
                // Create a directory for  to place the directory with the build results (if it does not exist)
                sh "mkdir -p ${resultdir}"
                git url: params.terracumber_gitrepo, branch: params.terracumber_ref
                dir("susemanager-ci") {
                    checkout scm
                }
            }

            stage('Name run') {
                currentBuild.description = nameDisplay(params)
            }

            stage('Build containers'){
                if (params.container_project && params.mi_project && params.mi_repo_name) {
                    def SCRIPT_DIR = "${WORKSPACE}/susemanager-ci/jenkins_pipelines/scripts/edit_bci_project"
                    sh "python3 -m venv ${WORKSPACE}/venv"
                    sh "${WORKSPACE}/venv/bin/pip install -r ${SCRIPT_DIR}/requirements.txt"
                    sh( script: "${WORKSPACE}/venv/bin/python ${SCRIPT_DIR}/edit.py --container-project ${params.container_project} --prefer ${params.prefer} --mi-project ${params.mi_project} --mi-repo-name ${params.mi_repo_name}", returnStdout: true)
                    custom_project_path = "registry.suse.de/${params.container_project.toLowerCase().replaceAll(':', '/')}/containerfile"
                    server_container_repository = custom_project_path
                    proxy_container_repository = custom_project_path
                }
            }

            stage('Deploy') {
                // Restore Terraform states from artifacts
                if (params.use_previous_terraform_state) {
                    copyArtifacts projectName: currentBuild.projectName, selector: specific("${currentBuild.previousBuild.number}")
                }

                if (params.must_deploy) {
                    // Clone sumaform
                    sh "set +x; source /home/jenkins/.credentials set -x; ./terracumber-cli ${common_params} --gitrepo ${params.sumaform_gitrepo} --gitref ${params.sumaform_ref} --runstep gitsync"

                    // Generate custom_repositories.json file in the workspace from the value passed by parameter
                    if (params.custom_repositories?.trim()) {
                        writeFile file: 'custom_repositories.json', text: params.custom_repositories, encoding: "UTF-8"
                    }
                    // Generate custom_repositories.json file in the workspace using a Python script - MI Identifiers passed by parameter
                    if (params.mi_ids?.trim()) {
                        node('manager-jenkins-node') {
                            checkout scm
                            def res_python_script_ = sh(script: "python3 jenkins_pipelines/scripts/json_generator/maintenance_json_generator.py --mi_ids ${params.mi_ids}", returnStatus: true)
                            echo "Build Validation JSON script return code:\n ${res_python_script_}"
                            if (res_python_script_ != 0) {
                                error("MI IDs (${params.mi_ids}) passed by parameter are wrong (or already released)")
                            }
                        }
                    }
                    // Run Terracumber to deploy the environment
                    sh """
                        set +x
                        source /home/jenkins/.credentials
                        set -x
                    
                        export TF_VAR_SERVER_CONTAINER_REPOSITORY=${server_container_repository}
                        export TF_VAR_PROXY_CONTAINER_REPOSITORY=${proxy_container_repository}
                        export TF_VAR_SERVER_CONTAINER_IMAGE=${server_container_image}
                        export TF_VAR_CUCUMBER_GITREPO=${params.cucumber_gitrepo}
                        export TF_VAR_CUCUMBER_BRANCH=${params.cucumber_ref}
                        export TF_VAR_PRODUCT_VERSION=${product_version}
                        export TF_VAR_BASE_OS=${base_os}
                        export TF_LOG=${params.extra_logs_level}
                        export TERRAFORM=${params.bin_path}
                        export TERRAFORM_PLUGINS=${params.bin_plugins_path}
                    
                        ./terracumber-cli ${common_params} \
                            --logfile ${resultdirbuild}/sumaform.log \
                            --init \
                            --taint '.*(domain|combustion_disk|cloudinit_disk|ignition_disk|main_disk|data_disk|database_disk|standalone_provisioning|server_extra_nfs_mounts).*' \
                            --custom-repositories ${WORKSPACE}/custom_repositories.json \
                            --sumaform-backend ${params.sumaform_backend} \
                            --use-tf-resource-cleaner \
                            --tf-resources-to-keep ${params.minions_to_run.split(', ').join(' ')} \
                            --runstep provision
                    """
                    // Generate features
                    runCucumberRakeTarget('utils:generate_build_validation_features')
                    // Generate rake files
                    runCucumberRakeTarget('jenkins:generate_rake_files_build_validation')
                    deployed = true
                }
            }

            stage('Sanity check') {
                def nodesHandler = getNodesHandler()
                runCucumberRakeTarget('cucumber:build_validation_sanity_check', false, nodesHandler.envVariableListToDisable)
                // Extract controller hostname
                try {
                    env.controller_hostname = sh(
                            script: """
                            set -e
                            cd ${localSumaformDirPath}
                            tofu output -json configuration | jq -r '.controller.hostname'
                        """,
                            returnStdout: true
                    ).trim()

                    // Print the values for confirmation
                    echo "Extracted controller hostname: ${env.controller_hostname}"

                } catch (Exception e) {
                    error("Failed to extract hostnames: ${e.message}")
                }
            }

            stage('Run core features') {
                if (params.must_run_core && (deployed || !params.must_deploy)) {
                    runCucumberRakeTarget('cucumber:build_validation_core')
                }
            }

            stage('Copy the new custom repository json file to controller') {


                if (params.push_new_custom_repositories) {
                    // Generate custom_repositories.json file in the workspace from the value passed by parameter
                    if (params.custom_repositories?.trim()) {
                        writeFile file: 'custom_repositories.json', text: params.custom_repositories, encoding: "UTF-8"
                    }

                    // Generate custom_repositories.json file in the workspace using a Python script - MI Identifiers passed by parameter
                    if (params.mi_ids?.trim()) {
                        node('manager-jenkins-node') {
                            checkout scm
                            def res_python_script_ = sh(script: "python3 jenkins_pipelines/scripts/json_generator/maintenance_json_generator.py --mi_ids ${params.mi_ids}", returnStatus: true)
                            echo "Build Validation JSON script return code:\n ${res_python_script_}"
                            if (res_python_script_ != 0) {
                                error("MI IDs (${params.mi_ids}) passed by parameter are wrong (or already released)")
                            }
                        }
                    }
                    sh(script: "${TestEnvironmentCleanerProgram} --url ${env.controller_hostname} --mode update_custom_repositories")
                }
            }

            stage('Sync. products and channels') {
                if (params.must_sync && (deployed || !params.must_deploy)) {
                    // Get minion list from tofu state list command
                    def nodesHandler = getNodesHandler()
                    res_products = runCucumberRakeTarget('cucumber:build_validation_reposync', true, nodesHandler.envVariableListToDisable)
                    echo "Custom channels and MU repositories status code: ${res_products}"
                    sh "exit ${res_products}"
                }
            }

            /** Proxy stages begin **/
            stage('Add MUs Proxy') {
                if (params.must_add_MU_repositories && params.enable_proxy_stages) {
                    echo 'Add proxy MUs'
                    if (params.confirm_before_continue) {
                        input 'Press any key to start adding Maintenance Update repositories'
                    }
                    echo 'Add custom channels and MU repositories'
                    res_mu_repos = runCucumberRakeTarget('cucumber:build_validation_add_maintenance_update_repositories_proxy', true)
                    echo "Custom channels and MU repositories status code: ${res_mu_repos}"
                    sh "exit ${res_mu_repos}"
                }
            }
            stage('Add Activation Keys Proxy') {
                if (params.must_add_keys && params.enable_proxy_stages) {
                    echo 'Add proxy activation key'
                    if (params.confirm_before_continue) {
                        input 'Press any key to start adding activation keys'
                    }
                    res_add_keys = runCucumberRakeTarget('cucumber:build_validation_add_activation_key_proxy',true)
                    echo "Add Proxy Activation Key status code: ${res_add_keys}"
                    sh "exit ${res_add_keys}"
                }
            }
            stage('Create bootstrap repository Proxy') {
                if (params.must_create_bootstrap_repos && params.enable_proxy_stages) {
                    echo 'Create bootstrap repository ${node}'
                    if (params.confirm_before_continue) {
                        input 'Press any key to start creating the proxy bootstrap repository'
                    }
                    res_create_bootstrap_repos = runCucumberRakeTarget('cucumber:build_validation_create_bootstrap_repository_proxy',true)
                    echo "Create Proxy bootstrap repository status code: ${res_create_bootstrap_repos}"
                    sh "exit ${res_create_bootstrap_repos}"
                }
            }
            stage('Bootstrap Proxy') {
                if (params.must_boot_node && params.enable_proxy_stages) {
                    if (params.confirm_before_continue) {
                        input 'Press any key to start bootstraping the Proxy'
                    }
                    res_init_proxy = runCucumberRakeTarget('cucumber:build_validation_init_proxy', true)
                    echo "Init Proxy status code: ${res_init_proxy}"
                    sh "exit ${res_init_proxy}"
                }
            }
            /** Proxy stages end **/

            /** Monitoring stages begin **/
            // Hide monitoring for qe update pipeline
            if (params.enable_monitoring_stages) {
                try {
                    stage('Add MUs Monitoring') {
                        if (params.must_add_MU_repositories && params.enable_monitoring_stages) {
                            if (params.confirm_before_continue) {
                                input 'Press any key to start adding Maintenance Update repositories'
                            }
                            echo 'Add custom channels and MU repositories'
                            res_mu_repos = runCucumberRakeTarget('cucumber:build_validation_add_maintenance_update_repositories_monitoring_server', true)
                            echo "Custom channels and MU repositories status code: ${res_mu_repos}"
                            sh "exit ${res_mu_repos}"
                        }
                    }
                    stage('Add Activation Keys Monitoring') {
                        if (params.must_add_keys && params.enable_monitoring_stages) {
                            echo 'Add server monitoring activation key'
                            if (params.confirm_before_continue) {
                                input 'Press any key to start adding activation keys'
                            }
                            res_add_keys = runCucumberRakeTarget('cucumber:build_validation_add_activation_key_monitoring_server', true)
                            echo "Add Server Monitoring Activation Key status code: ${res_add_keys}"
                            sh "exit ${res_add_keys}"
                        }
                    }
                    stage('Create bootstrap repository Monitoring') {
                        if (params.must_create_bootstrap_repos && params.enable_monitoring_stages) {
                            echo 'Create server monitoring bootstrap repository'
                            if (params.confirm_before_continue) {
                                input 'Press any key to start creating the Server Monitoring bootstrap repository'
                            }
                            res_create_bootstrap_repos = runCucumberRakeTarget('cucumber:build_validation_create_bootstrap_repository_monitoring_server', true)
                            echo "Create Server Monitoring bootstrap repository status code: ${res_create_bootstrap_repos}"
                            sh "exit ${res_create_bootstrap_repos}"
                        }
                    }
                    stage('Bootstrap Monitoring Server') {
                        if (params.must_boot_node && params.enable_monitoring_stages) {
                            if (params.confirm_before_continue) {
                                input 'Press any key to start bootstraping the Monitoring Server'
                            }
                            echo 'Register monitoring server as minion with gui'
                            res_init_monitoring = runCucumberRakeTarget('cucumber:build_validation_init_monitoring', true)
                            echo "Init Monitoring Server status code: ${res_init_monitoring}"
                            sh "exit ${res_init_monitoring}"
                        }
                    }
                } catch (Exception ex) {
                    println('Monitoring server bootstrap failed ')
                    monitoring_stage_result_fail = true
                }
            }
            /** Monitoring stages end **/

            /** Clients stages begin **/
            if (params.enable_client_stages) {
                // Call the minion testing.
                try {
                    stage('Clients stages') {
                        clientTestingStages(params)
                    }
                } catch (Exception ex) {
                    println('ERROR: one or more clients have failed')
                    client_stage_result_fail = true
                }
            }
            /** Clients stages end **/

            /** Products and Salt migration stages begin **/
            try {
                stage('Products and Salt migration stages') {
                    if(params.must_run_products_and_salt_migration_tests){
                        clientMigrationStages()
                    }
                }
            } catch (Exception ex) {
                println('ERROR: one or more migrations have failed')
                products_and_salt_migration_stage_result_fail = true
            }
            /** Products and Salt migration stages stages end **/

            /** Retail stages begin **/
            stage('Retail: Bootstrap build hosts') {
                if (params.must_prepare_retail) {
                    if (params.confirm_before_continue) {
                        input 'Press any key to start running the retail tests'
                    }
                    parallel (
                            'Init build host sles15sp7': {
                                stage('Init build host sles15sp7') {
                                    def res_init_buildhost_sles15sp7 = runCucumberRakeTarget('cucumber:build_validation_retail_init_sles15sp7_buildhost', true)
                                    echo "Retail proxy status code: ${res_init_buildhost_sles15sp7}"
                                    sh "exit ${res_init_buildhost_sles15sp7}"
                                }
                            },
                            'Init build host sles15sp6': {
                                stage('Init build host sles15sp6') {
                                    def res_init_buildhost_sles15sp6 = runCucumberRakeTarget('cucumber:build_validation_retail_init_sles15sp6_buildhost', true)
                                    echo "Retail proxy status code: ${res_init_buildhost_sles15sp6}"
                                    sh "exit ${res_init_buildhost_sles15sp6}"
                                }
                            }
                    )
                }
            }
            stage('Retail: Test terminal deployments') {
                if (params.must_test_retail_terminal) {
                    if (params.confirm_before_continue) {
                        input 'Press any key to start running the retail tests'
                    }
                    // ----- Start: Get Terminal List -----
                    // Dynamically create the terminal list to test depending on the state list
                    Set<String> terminalsList = new HashSet<String>()
                    def tfState = sh(script: "cd ${resultdir}/sumaform; tofu state list", returnStdout: true)
                    String[] moduleList = tfState.split("\n")
                    moduleList.each { lane ->
                        def instanceList = lane.tokenize(".")
                        if (instanceList[1].contains('terminal')) {
                            terminalsList.add(instanceList[1].replace('_terminal', ''))
                        }
                    }
                    echo "Dynamic Terminal List detected from State: ${terminalsList}"
                    if (terminalsList.isEmpty()) {
                        error "No terminal modules found in Terraform state! Expected format: module.<name>_terminal..."
                    }
                    // ----- End: Get Terminal List -----

                    def terminal_deployment_testing = [:]
                    // PENDING = Not run yet
                    // SUCCESS = Configured successfully
                    // FAILURE = Configured failed, do not retry
                    def retailProxyStatus = [status: 'PENDING']
                    terminalsList.each { terminal ->
                        terminal_deployment_testing["${terminal}"] = {
                            stage("Build image for ${terminal}") {
                                def res_build_image = runCucumberRakeTarget("cucumber:build_validation_retail_build_image_${terminal}", true)
                                sh "exit ${res_build_image}"situations
                            }
                            // TODO: Move back configure retail proxy to Retail: Bootstrap build hosts stage once 4.3 and 5.0 are EOL
                            // Need to be executed after building images for 5.0
                            // Using lock and proxyHandler to make sure to run it only once, first to start.
                            stage("Configure retail proxy (${terminal})") {
                                lock(resource: retailProxyConfigurationLock) {
                                    if (retailProxyStatus['status'] == 'FAILURE') {
                                        error "Aborting ${terminal}: Retail proxy configuration failed by another branch."
                                    } else if (retailProxyStatus['status'] == 'SUCCESS') {
                                        echo "Configure retail proxy already completed (skipped for ${terminal})"
                                    } else {
                                        def res_configure_retail_proxy = runCucumberRakeTarget('cucumber:build_validation_retail_configure_proxy', true)
                                        if (res_configure_retail_proxy != 0) {
                                            // CRITICAL: Mark as FAILURE before throwing error so other waiting threads see it
                                            retailProxyStatus['status'] = 'FAILURE'
                                            error "Retail proxy configuration failed with exit code: ${res_configure_retail_proxy}"
                                        } else {
                                            // Mark as SUCCESS
                                            retailProxyStatus['status'] = 'SUCCESS'
                                            echo "Proxy successfully configured by ${terminal}"
                                        }
                                    }
                                }
                            }
                            stage("Prepare group and saltboot for ${terminal}") {
                                def res_prepare_group_saltboot = runCucumberRakeTarget("cucumber:build_validation_retail_prepare_group_saltboot_${terminal}", true)
                                sh "exit ${res_prepare_group_saltboot}"
                            }
                            stage("Deploy terminal ${terminal}") {
                                def res_deploy_terminal = runCucumberRakeTarget("cucumber:build_validation_retail_deploy_terminal_${terminal}", true)
                                sh "exit ${res_deploy_terminal}"
                            }
                        }
                    }
                    parallel terminal_deployment_testing
                }
            }
            /** Retail stages end **/

            /** Containerization stages start **/
            try {
                stage('Containerization') {
                    if (params.must_run_containerization_tests ?: false) {
                        if (params.confirm_before_continue) {
                            input 'Press any key to start running the containerization tests'
                        }
                        echo 'Prepare Proxy as Pod and run basic tests'
                        def res_container_proxy = runCucumberRakeTarget('cucumber:build_validation_containerization', true)
                        echo "Container proxy status code: ${res_container_proxy}"
                        if (res_container_proxy != 0) {
                            error("Containerization test failed with status code: ${res_container_proxy}")
                        }
                    }
                }
            } catch (Exception ex) {
                println('ERROR: Containerization failed')
                containerization_stage_result_fail = true
            }
            /** Containerization stages end **/
        }
        finally {
            stage('Save TF state') {
                archiveArtifacts artifacts: "results/sumaform/terraform.tfstate, results/sumaform/.terraform/**/*"
            }

            stage('Get results') {
                def result_error = 0
                if (deployed || !params.must_deploy) {
                    try {
                        runCucumberRakeTarget('cucumber:build_validation_finishing')
                    } catch(Exception ex) {
                        println("ERROR: rake cucumber:build_validation_finishing failed")
                        result_error = 1
                    }
                    try {
                        runCucumberRakeTarget('utils:generate_test_report')
                    } catch(Exception ex) {
                        println("ERROR: rake utils:generate_test_report failed")
                        result_error = 1
                    }
                    sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep getresults"
                    publishHTML( target: [
                            allowMissing: true,
                            alwaysLinkToLastBuild: false,
                            keepAll: true,
                            reportDir: "${resultdirbuild}/cucumber_report/",
                            reportFiles: 'cucumber_report.html',
                            reportName: "Build Validation report"]
                    )
//                    junit allowEmptyResults: true, testResults: "${junit_resultdir}/*.xml"
                }
                // Send email
                sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/mail.log --runstep mail"
                // Clean up old results
                sh "./clean-old-results -r ${resultdir}"
                // Fail pipeline if client stages failed
                if (client_stage_result_fail) {
                    error("Client stage failed")
                }
                // Fail pipeline if monitoring stages failed
                if (monitoring_stage_result_fail) {
                    error("Monitoring stage failed")
                }
                // Fail pipeline if products or Salt migration stages failed
                if (products_and_salt_migration_stage_result_fail) {
                    error("Product or Salt migration stage failed")
                }
                // Fail pipeline if retail stages failed
                if (retail_stage_result_fail) {
                    error("Retail stage failed")
                }
                // Fail pipeline if containerization stage failed
                if (containerization_stage_result_fail) {
                    error("Containerization stage failed")
                }
                sh "exit ${result_error}"
            }
        }
    }
}

/**
 * Executes a rake command inside the terracumber cucumber step.
 * @param rake_target The full rake target string (e.g., 'cucumber:build_validation_core').
 * @param disableMinions Optional list of environment variables to unset (for parallel client stages).
 * @param return_status Boolean to decide if the command should return the exit status.
 */
def runCucumberRakeTarget(String rake_target, boolean return_status = false, disableMinions = null) {
    // Note: The disableMinions is provided as a space-separated string in the code (e.g., in getNodesHandler()),
    // For compatibility with the original structure where getNodesHandler().envVariableListToDisable is a string.
    def unset_vars = ""
    if (disableMinions) {
        // If it's a list/set, join it to a string. If it's already a string, use it.
        def list_to_join = disableMinions instanceof String ? disableMinions.split(' ') : disableMinions
        unset_vars = list_to_join ? "unset ${list_to_join.join(' ')}; " : ""
    }

    def script = """
        ./terracumber-cli ${common_params} \\
            --logfile ${resultdirbuild}/testsuite.log \\
            --runstep cucumber \\
            --cucumber-cmd '${unset_vars}${env.exports} cd /root/spacewalk/testsuite; rake ${rake_target}'
    """

    // Remove leading/trailing whitespace and execute
    def final_script = script.stripIndent().trim()

    if (return_status) {
        return sh(script: final_script, returnStatus: true)
    } else {
        sh final_script
    }
}

// Develop a function that outlines the various stages of a minion.
// These stages will be executed concurrently.
def clientTestingStages(params) {

    // Implement a hash map to store the various stages of nodes.
    def tests = [:]

    // Load JSON matching non MU repositories data (non_MU_channels_tasks_file variable declared at the pipeline description level)
    def json_matching_non_MU_data = readJSON(file: params.non_MU_channels_tasks_file)

    //Get minion list from tofu state list command
    def nodesHandler = getNodesHandler()
    def bootstrap_repository_status = nodesHandler.BootstrapRepositoryStatus
    def required_custom_channel_status = nodesHandler.CustomChannelStatus
    // Construct a stage list for each node.
    nodesHandler.nodeList.each { node ->
        tests["${node}"] = {
            // Generate a temporary list that comprises of all the minions except the one currently undergoing testing.
            // This list is utilized to establish an SSH session exclusively with the minion undergoing testing.
            def temporaryList = nodesHandler.envVariableList.toList() - node.replaceAll('sles', 'sle').toUpperCase()
            def nodeTag = node.replace('sles1','sle1')
            stage("${node}") {
                echo "Testing ${node}"
            }
            stage("Add MUs ${node}") {
                if (params.must_add_MU_repositories) {
                    if (!node.contains('ssh')) {
                        if (params.confirm_before_continue) {
                            input 'Press any key to start adding Maintenance Update repositories'
                        }
                        echo 'Add custom channels and MU repositories'
                        res_mu_repos = runCucumberRakeTarget("cucumber:build_validation_add_maintenance_update_repositories_${nodeTag}", true, temporaryList)
                        echoHtmlReportPath("build_validation_add_maintenance_update_repositories_${nodeTag}")
                        echo "Custom channels and MU repositories status code: ${res_mu_repos}"
                        if (res_mu_repos != 0) {
                            required_custom_channel_status[node] = 'FAIL'
                            error("Add custom channels and MU repositories failed with status code: ${res_mu_repos}")
                        }
                    }
                }
                // Don't update required_custom_channel_status for minion who needs non MU repositories
                if (!json_matching_non_MU_data.containsKey(node)) {
                    required_custom_channel_status[node] = 'CREATED'
                }
            }
            stage("Add non MU Repositories ${node}") {
                if (params.must_add_non_MU_repositories) {
                    if (!node.contains('ssh')) {
                        // We have this condition inside the stage to see in Jenkins which minion is skipped
                        if (json_matching_non_MU_data.containsKey(node)) {
                            def build_validation_non_MU_script = json_matching_non_MU_data["${node}"]
                            if (params.confirm_before_continue) {
                                input 'Press any key to start adding common channels'
                            }
                            echo 'Add non MU Repositories'
                            res_non_MU_repositories = runCucumberRakeTarget("cucumber:${build_validation_non_MU_script}", true, temporaryList)
                            echo "Non MU Repositories status code: ${res_non_MU_repositories}"
                            echoHtmlReportPath(build_validation_non_MU_script)
                            if (res_non_MU_repositories != 0) {
                                required_custom_channel_status[node] = 'FAIL'
                                error("Add common channels failed with status code: ${res_non_MU_repositories}")
                            }
                        }
                    }
                }
                // Don't overwrite required_custom_channel_status variable for minions who don't need non MU repositories ( protect overwrite of add MU fails )
                if (json_matching_non_MU_data.containsKey(node)) {
                    required_custom_channel_status[node] = 'CREATED'
                }
            }
            stage("Add Activation Keys ${node}") {
                // skip this stage for Salt migration minion
                if (params.must_add_keys && !node.contains('salt_migration_minion')) {
                    if (node.contains('sshminion')) {
                        // SSH minion need mandatory custom channel repository. The channel is created during minion stage.
                        // This section wait until minion creates custom channel.
                        def minion_name_without_ssh = node.replaceAll('sshminion', 'minion')
                        println "Waiting for mandatory custom channel for ${node} to be created by ${minion_name_without_ssh}."
                        waitUntil {
                            required_custom_channel_status[minion_name_without_ssh] != 'NOT_CREATED'
                        }
                        if (required_custom_channel_status[minion_name_without_ssh] == 'FAIL') {
                            error("${minion_name_without_ssh} creates bootstrap repository failed")
                        }
                    }
                    if (params.confirm_before_continue) {
                        input 'Press any key to start adding activation keys'
                    }
                    echo 'Add Activation Keys'
                    res_add_keys = runCucumberRakeTarget("cucumber:build_validation_add_activation_key_${nodeTag}", true, temporaryList)
                    echoHtmlReportPath("build_validation_add_activation_key_${nodeTag}")
                    echo "Add Activation Keys status code: ${res_add_keys}"
                    if (res_add_keys != 0) {
                        bootstrap_repository_status[node] = 'FAIL'
                        error("Add Activation Keys failed with status code: ${res_add_keys}")
                    }
                }
            }
            stage("Create bootstrap repository ${node}") {
                if (params.must_create_bootstrap_repos) {
                    if (node.contains('sshminion')) {
                        // SSH minion need bootstrap repository. The bootstrap repository is created during minion stage.
                        // This section wait until minion creates bootstrap repository
                        def minion_name_without_ssh = node.replaceAll('sshminion', 'minion')
                        println "Waiting for bootstrap repository creation by ${minion_name_without_ssh} for ${node}."
                        waitUntil {
                            bootstrap_repository_status[minion_name_without_ssh] != 'NOT_CREATED'
                        }
                        if (bootstrap_repository_status[minion_name_without_ssh] == 'FAIL') {
                            error("${minion_name_without_ssh} creates bootstrap repository failed")
                        } else {
                            println "Bootstrap repository available for ${node} "
                        }
                    } else {
                        if (params.confirm_before_continue) {
                            input 'Press any key to start creating bootstrap repositories'
                        }
                        // Make sure s390 minions create their bootstrap repositories after the x86_64 equivalents to avoid flushing the repository folders.
                        if (node.contains('s390')){
                            def minion_name_without_s390 = node.replaceAll('s390', '')
                            waitUntil {
                                bootstrap_repository_status[minion_name_without_s390] != 'NOT_CREATED'
                            }
                        }
                        // Employ a lock resource to prevent concurrent calls to create the bootstrap repository in the manager.
                        // Utilize a try-catch mechanism to release the resource for other nodes in the event of a failed bootstrap.
                        lock(resource: mgrCreateBootstrapRepo, timeout: 320) {
                            try {
                                echo 'Create bootstrap repository'
                                res_create_bootstrap_repository = runCucumberRakeTarget("cucumber:build_validation_create_bootstrap_repository_${nodeTag}", true, temporaryList)
                                echoHtmlReportPath("build_validation_create_bootstrap_repository_${nodeTag}")
                                echo "Create bootstrap repository status code: ${res_create_bootstrap_repository}"
                                if (res_create_bootstrap_repository != 0) {
                                    bootstrap_repository_status[node] = 'FAIL'
                                    error("Create bootstrap repository failed with status code: ${res_create_bootstrap_repository}")
                                }
                            } finally {
                                echo 'Release resource mgrCreateBootstrapRepo'
                            }
                        }
                    }
                }
                bootstrap_repository_status[node] = 'CREATED'
            }
            stage("Bootstrap client ${node}") {
                if (params.must_boot_node) {
                    if (params.confirm_before_continue) {
                        input 'Press any key to start bootstraping the clients'
                    }
                    randomWait()
                    echo 'Bootstrap clients'
                    // Temporarily modify env.exports to include the bootstrap timeout
                    def custom_exports = "${env.exports} export DEFAULT_TIMEOUT=${env.bootstrap_timeout};"
                    // The helper doesn't easily allow overriding env.exports. For this unique case, we'll keep the logic inline to modify the environment variable within the script block for the special DEFAULT_TIMEOUT.
                    res_init_clients = sh(script: "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'unset ${temporaryList.join(' ')}; ${custom_exports} cd /root/spacewalk/testsuite; rake cucumber:build_validation_init_client_${nodeTag}'", returnStatus: true)
                    echoHtmlReportPath( "build_validation_init_client_${nodeTag}")
                    echo "Init clients status code: ${res_init_clients}"
                    if (res_init_clients != 0) {
                        error("Bootstrap clients failed with status code: ${res_init_clients}")
                    }
                }
            }
            stage("Run Smoke Tests ${node}") {
                if (params.must_run_tests) {
                    if (params.confirm_before_continue) {
                        input 'Press any key to start running the smoke tests'
                    }
                    randomWait()
                    echo 'Run Smoke tests'
                    res_smoke_tests = runCucumberRakeTarget("cucumber:build_validation_smoke_tests_${nodeTag}", true, temporaryList)
                    echoHtmlReportPath("build_validation_smoke_tests_${nodeTag}")
                    echo "Smoke tests status code: ${res_smoke_tests}"
                    if (res_smoke_tests != 0) {
                        error("Run Smoke tests failed with status code: ${res_smoke_tests}")
                    }
                }
            }
        }
    }
    // Once all the stages have been correctly configured, run in parallel
    parallel tests
}

def getNodesHandler() {
    // Employ the terraform state list command to generate the list of nodes.
    // Due to the disparity between the node names in the test suite and those in the environment variables of the controller, two separate lists are maintained.
    Set<String> nodeList = new HashSet<String>()
    Set<String> envVar = new HashSet<String>()
    def BootstrapRepositoryStatus = [:]
    def CustomChannelStatus       = [:]
    modules = sh(script: "cd ${resultdir}/sumaform; tofu state list",
            returnStdout: true)
    String[] moduleList = modules.split("\n")
    moduleList.each { lane ->
        def instanceList = lane.tokenize(".")
        if (instanceList[1].contains('minion') || instanceList[1].contains('client')) {
            nodeList.add(instanceList[1])
            envVar.add(instanceList[1].replaceAll('sles', 'sle').toUpperCase())
        }
    }
    // Convert jenkins minions list parameter to list
    Set<String> nodesToRun = params.minions_to_run.split(', ')
    // Create a variable with declared nodes on Jenkins side but not deploy and print it
    def notDeployedNode = nodesToRun.findAll { !nodeList.contains(it) }
    println "These minions are declared in jenkins but not deployed! ${notDeployedNode}"
    // Check the difference between the nodes deployed and the nodes declared in Jenkins side
    // This difference will be the nodes to disable
    def disabledNodes = nodeList.findAll { !nodesToRun.contains(it) }
    // Convert this list to cucumber compatible environment variable
    def envVarDisabledNodes = disabledNodes.collect { it.replaceAll('sles', 'sle').toUpperCase() }
    // Create a node list without the disabled nodes. ( use to configure the client stage )
    def nodeListWithDisabledNodes = nodeList - disabledNodes
    // Create a map storing mu synchronization state for each minion.
    // This map is to be sure ssh minions have the MU channel ready.
    for (node in nodeListWithDisabledNodes ) {
        BootstrapRepositoryStatus[node] = 'NOT_CREATED'
        CustomChannelStatus[node]       = 'NOT_CREATED'
    }
    // envVariableListToDisable is returned as a space-separated string, as it's used directly in the 'sh' script in 'Sanity check'
    return [nodeList:nodeListWithDisabledNodes, envVariableList:envVar, envVariableListToDisable:envVarDisabledNodes.join(' '), CustomChannelStatus:CustomChannelStatus, BootstrapRepositoryStatus:BootstrapRepositoryStatus]
}

// Creates a stage for each product or client migration feature.
// Proceeds to execute them concurrently.
def clientMigrationStages() {
    def migration_tests = [:]

    features_list = sh(script: "./terracumber-cli ${common_params} --runstep cucumber --cucumber-cmd 'ls -1 /root/spacewalk/testsuite/features/build_validation/migration/'", returnStdout: true)
    String[] migration_features = features_list.split("\n").findAll { it.contains("migration_") }
    // create a stage for each migration feature
    migration_features.each{ feature ->
        def minion = cleanMigrationFeatureName(feature)

        migration_tests["${minion}"] = {
            if (params.confirm_before_continue) {
                input "Press any key to start testing the migration of ${minion}"
            }
            stage("${minion} migration") {
                res_minion_migration = runCucumberRakeTarget("cucumber:build_validation_migration_${minion}", true)
                echo "${minion} migration status code: ${res_minion_migration}"
                if (res_minion_migration != 0) {
                    error("Migration test for ${minion} failed with status code: ${res_minion_migration}")
                }
            }
        }
    }
    // run them in parallel
    parallel migration_tests
}

def randomWait() {
    def randomWait = new Random().nextInt(180)
    println "Waiting for ${randomWait} seconds"
    sleep randomWait
}

def nameDisplay(params) {
    def buildLabel = []

    if (params.must_deploy) buildLabel << 'deploy'
    if (params.must_run_core) buildLabel << 'core'
    if (params.must_sync) buildLabel << 'reposync'

    buildLabel << buildComponentLabel("proxy", params, [
            'must_add_MU_repositories'     : 'AddMU',
            'must_add_keys'                : 'ActKeys',
            'must_create_bootstrap_repos'  : 'CrBoot',
            'must_boot_node'               : 'Boot'
    ], params.enable_proxy_stages)

    buildLabel << buildComponentLabel("monitoring", params, [
            'must_add_MU_repositories'     : 'AddMU',
            'must_add_keys'                : 'ActKeys',
            'must_create_bootstrap_repos'  : 'CrBoot',
            'must_boot_node'               : 'Boot'
    ], params.enable_monitoring_stages)

    buildLabel << buildComponentLabel("client", params, [
            'must_add_MU_repositories'     : 'AddMU',
            'must_add_non_MU_repositories' : 'AddNonMU',
            'must_add_keys'                : 'ActKeys',
            'must_create_bootstrap_repos'  : 'CrBoot',
            'must_boot_node'               : 'Boot',
            'must_run_tests'               : 'Smoke'
    ], params.enable_client_stages)

    if (params.must_run_products_and_salt_migration_tests) buildLabel << 'migration'
    if (params.must_prepare_retail) buildLabel << 'PrepRetail'
    if (params.must_test_retail_terminal) buildLabel << 'TestRetail'

    // Manually filter null/empty items to avoid findAll()
    def filteredLabel = []
    for (item in buildLabel) {
        if (item) {
            filteredLabel << item
        }
    }

    def baseOs = params.base_os ?: ''
    def fullLabel = "${params.product_version_display}${baseOs ? "_${baseOs}" : ""} - ${filteredLabel.join(' ')}"

    if (fullLabel.length() > 160) {
        return "#${env.BUILD_NUMBER} - ${params.product_version_display}${baseOs ? "_${baseOs}" : ""}"
    }
    return fullLabel
}

def buildComponentLabel(component, params, conditionsMap, isEnabled) {
    if (!isEnabled) return null

    def options = []
    for (entry in conditionsMap) {
        if (params.get(entry.key)) {
            options << entry.value
        }
    }
    return options ? "${component}[${options.join(' ')}]" : null
}

// Function to extract the minion name from the migration feature
def cleanMigrationFeatureName(String feature) {
    if (feature.startsWith("migration_")) {
        feature = feature.substring("migration_".length())
    }
    if (feature.endsWith(".feature")) {
        feature = feature.substring(0, feature.length() - ".feature".length())
    }
    return feature
}

/**
 * Fetches the HTML report path from the file exposed via HTTP on the controller
 * and logs the full HTTP link to the job steps.
 * * @param rake_target The rake target (e.g., build_validation_smoke_tests_NODETAG).
 */
def echoHtmlReportPath(String rake_target) {
    // Construct the full URL to the file containing the HTML path
    def path_export_url = "http://${env.controller_hostname}/results/${env.BUILD_NUMBER}/${rake_target}_html_path.txt"

    def html_report_path = ''
    try {
        // Use httpRequest to fetch the file content
        def response = httpRequest(url: path_export_url, throwExceptionOnError: true, quiet: true)

        if (response.status == 200) {
            // The content is the full relative path to the HTML report
            html_report_path = response.content.trim()

            // Construct and log the full HTTP URL for the actual report
            def full_html_url = "http://${env.controller_hostname}/${html_report_path}"
            echo "HTML Report URL: ${full_html_url}"
        } else {
            echo "Warning: Failed to fetch HTML path file. HTTP Status: ${response.status}"
        }
    } catch (Exception e) {
        // This catches network errors, DNS failures, or httpRequest throwing
        // an exception if throwExceptionOnError is true (e.g., 404 response).
        echo "Error fetching HTML path from ${path_export_url}: ${e.getMessage()}"
    }
}

return this