Add parallel argument to terraform gluescript command (#346)

Add --parallel to be able to control inner terraform -parallelism=N in
plan and apply
commit also include UT and e2e test. In particular the e2e test has a
debug output showing the effect of the parallelism within terraform
itself.

Author: mpagot

scripts/qesap/lib/cmds.py

@@ -165,15 +165,17 @@ def cmd_destroy(configure_data, base_project, dryrun=False, verbose=False):
     return cmd_terraform(configure_data, base_project, dryrun, workspace='default', destroy=True)
 
 
-def cmd_terraform(configure_data, base_project, dryrun, workspace='default', destroy=False):
+def cmd_terraform(configure_data, base_project, dryrun, workspace='default', destroy=False, parallel=None):
     """ Main executor for the deploy sub-command
 
     Args:
         configure_data (obj): configuration structure
         base_project (str): base project path where to
                       look for the Terraform files
-        destroy (bool): destroy
         dryrun (bool): enable dryrun execution mode
+        workspace (str): name of the workspace to activate before running the deployment
+        destroy (bool): destroy
+        parallel (int): value to use for argument --parallelism=n when call terraform plan and apply
 
     Returns:
         Status: execution result, 0 means OK. It is mind to be used as script exit code
@@ -199,8 +201,11 @@ def cmd_terraform(configure_data, base_project, dryrun, workspace='default', des
         cmds.append(f"{terraform_common_cmd} init")
         if workspace != 'default':
             cmds.append(f"{terraform_common_cmd} workspace new {workspace}")
-        cmds.append(f"{terraform_common_cmd} plan -out=plan.zip")
-        cmds.append(f"{terraform_common_cmd} apply -auto-approve plan.zip")
+        parallel_str = ''
+        if parallel:
+            parallel_str = f"-parallelism={parallel} "
+        cmds.append(f"{terraform_common_cmd} plan {parallel_str}-out=plan.zip")
+        cmds.append(f"{terraform_common_cmd} apply {parallel_str}-auto-approve plan.zip")
 
     for command in cmds:
         command += ' -no-color'

scripts/qesap/qesap.py

@@ -114,6 +114,11 @@ def cli(command_line=None):
                                   dest='workspace',
                                   default='default',
                                   help="""Workspace to use in terraform commands. Defaults to 'default'""")
+    parser_terraform.add_argument('-p',
+                                  '--parallel',
+                                  type=int,
+                                  dest='parallel',
+                                  help="""Set value for -parallelism for plan and apply""")
     parser_ansible = subparsers.add_parser('ansible', help="Run the Ansible part of the deployment")
     parser_ansible.add_argument('-d',
                                 '--destroy',
@@ -182,7 +187,8 @@ def main(command_line=None):  # pylint: disable=too-many-return-statements
             parsed_args.basedir,
             parsed_args.dryrun,
             workspace=parsed_args.workspace,
-            destroy=parsed_args.destroy
+            destroy=parsed_args.destroy,
+            parallel=parsed_args.parallel
         )
         if res != 0:
             print(res.msg)  # should we use file=sys.stderr here?

scripts/qesap/test/e2e/main_local_many.tf

@@ -0,0 +1,10 @@
+resource "local_file" "foo" {
+  # 10 is the default parallel level in terraform when not using -parallelism=N
+  # so use it to maximise the difference in using or not using the argument
+  count    = 10
+  content  = "@${timestamp()} foo@"
+  filename = "${path.module}/foo.${count.index}.bar"
+  provisioner "local-exec" {
+    command = "sleep 2 ; date +'%T %N'"
+  }
+}

scripts/qesap/test/e2e/test.sh

@@ -375,7 +375,47 @@ lines=$(cat "${THIS_LOG}" | wc -l)
 grep -E "SOMETHING INVALID" $THIS_LOG || test_die "Expected content not found in ${THIS_LOG}"
 rm terraform.*.log.txt
 rm "${TEST_PROVIDER}/main.tf"
-touch "${TEST_PROVIDER}/main.tf"
+
+#######################################################################
+test_step "[${QESAP_CFG}] test parallel dryrun"
+
+THIS_LOG="${QESAPROOT}/test_parallel.txt"
+reset_root
+cp main_local_many.tf "${TEST_PROVIDER}/main.tf"
+qesap.py --verbose -b ${QESAPROOT} -c ${QESAP_CFG} --dryrun terraform --parallel 1 |& tee "${THIS_LOG}"
+grep -E "plan.*-parallelism=1" ${THIS_LOG} || test_die "Missing argument -parallelism=1 in terraform plan"
+grep -E "apply.*-parallelism=1" ${THIS_LOG} || test_die "Missing argument -parallelism=1 in terraform apply"
+rm ${THIS_LOG}
+
+#######################################################################
+test_step "[${QESAP_CFG}] test parallel"
+
+# run reference test without parallel
+reset_root
+cp main_local_many.tf "${TEST_PROVIDER}/main.tf"
+qesap.py --verbose -b ${QESAPROOT} -c ${QESAP_CFG} terraform |& tee "${THIS_LOG}"
+grep local-exec ${THIS_LOG}
+for foofile in "${TEST_PROVIDER}"/foo.*; do
+  printf "#${foofile} --> $(cat ${foofile})#\n"
+done
+count=$(grep -rhE "^@.*foo@" "${TEST_PROVIDER}" | sort -u | wc -l)
+[[ $count -eq 1 ]] || test_die "Not all the generated files has the same content. Count:${count}"
+
+qesap.py --verbose -b ${QESAPROOT} -c ${QESAP_CFG} terraform -d
+rm ${THIS_LOG}
+
+test_split
+
+reset_root
+cp main_local_many.tf "${TEST_PROVIDER}/main.tf"
+qesap.py --verbose -b ${QESAPROOT} -c ${QESAP_CFG} terraform --parallel 1 |& tee "${THIS_LOG}"
+grep local-exec ${THIS_LOG}
+for foofile in "${TEST_PROVIDER}"/foo.*; do
+  printf "#${foofile} --> $(cat ${foofile})#\n"
+done
+count=$(grep -rhE "^@.*foo@" "${TEST_PROVIDER}" | sort -u | wc -l)
+[[ $count -ne 1 ]] || test_die "All the generated files has the same content. Count:${count}"
+rm ${THIS_LOG}
 
 
 echo "#######################################################################"

scripts/qesap/test/unit/test_qesap_terraform.py

@@ -252,3 +252,24 @@ def test_terraform_call_terraform_workspace_destroy(subprocess_run, args_helper,
     calls.append(mock.call(f"terraform -chdir={terraform_dir} workspace delete lucignolo -no-color"))
 
     subprocess_run.assert_has_calls(calls)
+
+
+@mock.patch("lib.process_manager.subprocess_run")
+def test_terraform_call_terraform_parallel(subprocess_run, args_helper, config_yaml_sample):
+    """
+    Command terraform calls 'terraform plan' and 'terraform apply' with -parallelism=n
+    """
+    provider = 'mangiafuoco'
+    conf = config_yaml_sample(provider)
+
+    args, terraform_dir, *_ = args_helper(provider, conf)
+    args.extend(['terraform', '-p', '5'])
+    subprocess_run.return_value = (0, [])
+    assert main(args) == 0
+    subprocess_run.assert_called()
+
+    calls = []
+    calls.append(mock.call(f"terraform -chdir={terraform_dir} plan -parallelism=5 -out=plan.zip -no-color"))
+    calls.append(mock.call(f"terraform -chdir={terraform_dir} apply -parallelism=5 -auto-approve plan.zip -no-color"))
+
+    subprocess_run.assert_has_calls(calls)