Merge pull request #404 from rundeck-plugins/issue_rpl-67_ansible-max-aliases

RPL-67: Fix - Added max aliases field to set a valid value

Author: alexander-variacode

functional-test/src/test/groovy/functional/MaxAliasesSpec.groovy

@@ -0,0 +1,39 @@
+package functional
+
+import functional.base.BaseTestConfiguration
+import org.testcontainers.spock.Testcontainers
+
+@Testcontainers
+class MaxAliasesSpec extends BaseTestConfiguration {
+
+    static String PROJ_NAME = 'ansible-max-aliases'
+    static String NODE_1 = 'proxy-1.example.net'
+    static String NODE_2 = 'proxy-2.example.net'
+    static String NODE_3 = 'proxy-3.example.net'
+
+    def setupSpec() {
+        startCompose()
+        configureRundeck(PROJ_NAME, NODE_1)
+    }
+
+    void "max aliases"() {
+        when:
+        def result = client.apiCall {api-> api.listNodes(PROJ_NAME,'.*')}
+
+        then:
+        result != null
+        result.size() == 4
+        result.get(NODE_1) != null
+        result.get(NODE_1).getAttributes().get('nodename') == NODE_1
+        result.get(NODE_1).getAttributes().get('hostname') == NODE_1
+        result.get(NODE_1).getAttributes().get('tags') == 'fr, fr1'
+        result.get(NODE_2) != null
+        result.get(NODE_2).getAttributes().get('nodename') == NODE_2
+        result.get(NODE_2).getAttributes().get('hostname') == NODE_2
+        result.get(NODE_2).getAttributes().get('tags') == 'fr, fr1'
+        result.get(NODE_3) != null
+        result.get(NODE_3).getAttributes().get('nodename') == NODE_3
+        result.get(NODE_3).getAttributes().get('hostname') == NODE_3
+        result.get(NODE_3).getAttributes().get('tags') == 'fr2'
+    }
+}

functional-test/src/test/resources/docker/ansible-max-aliases/ansible.cfg

@@ -0,0 +1,6 @@
+[defaults]
+inventory=/home/rundeck/ansible-max-aliases/inventory_max_aliases_52.ini
+interpreter_python=/usr/bin/python3
+
+
+

functional-test/src/test/resources/docker/ansible-max-aliases/inventory_max_aliases_52.ini

@@ -0,0 +1,38 @@
+[fr:children]
+fr1
+fr2
+
+[fr:vars]
+api_dcs=["fr1","fr2"]
+alias_01=["data-01","data-02"]
+alias_02=["data-01","data-02"]
+alias_03=["data-01","data-02"]
+alias_04=["data-01","data-02"]
+alias_05=["data-01","data-02"]
+alias_06=["data-01","data-02"]
+alias_07=["data-01","data-02"]
+alias_08=["data-01","data-02"]
+alias_09=["data-01","data-02"]
+alias_10=["data-01","data-02"]
+alias_11=["data-01","data-02"]
+alias_12=["data-01","data-02"]
+alias_13=["data-01","data-02"]
+alias_14=["data-01","data-02"]
+alias_15=["data-01","data-02"]
+alias_16=["data-01","data-02"]
+alias_17=["data-01","data-02"]
+alias_18=["data-01","data-02"]
+alias_19=["data-01","data-02"]
+alias_20=["data-01","data-02"]
+alias_21=["data-01","data-02"]
+alias_22=["data-01","data-02"]
+alias_23=["data-01","data-02"]
+alias_24=["data-01","data-02"]
+alias_25=["data-01","data-02"]
+
+[fr1]
+proxy-1.example.net name=proxy-1 peers_ip=10.3.13.221 mgmt_ip=10.3.16.221 is_mon_master=True
+proxy-2.example.net name=proxy-2 peers_ip=10.3.13.222 mgmt_ip=10.3.16.222 is_mon_master=True
+
+[fr2]
+proxy-3.example.net name=proxy-3 peers_ip=10.3.13.223 mgmt_ip=10.3.16.223 is_mon_master=True

functional-test/src/test/resources/docker/docker-compose.yml

@@ -36,6 +36,7 @@ services:
     - ./ansible-list:/home/rundeck/ansible-list:rw
     - ./ansible-yaml-parsing:/home/rundeck/ansible-yaml-parsing:rw
     - ./ansible-child-groups:/home/rundeck/ansible-child-groups:rw
+    - ./ansible-max-aliases:/home/rundeck/ansible-max-aliases:rw
 
 volumes:
   rundeck-data:

functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/acls/node-acl.aclpolicy

@@ -0,0 +1,8 @@
+by:
+  urn: project:ansible-yaml-parsing
+for:
+  storage:
+    - match: 
+        path: 'keys/.*'
+      allow: [read] 
+description: Allow access to key storage

functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/etc/project.properties

@@ -0,0 +1,32 @@
+#edit below
+project.disable.executions=false
+project.disable.schedule=false
+project.execution.history.cleanup.batch=500
+project.execution.history.cleanup.enabled=false
+project.execution.history.cleanup.retention.days=60
+project.execution.history.cleanup.retention.minimum=50
+project.execution.history.cleanup.schedule=0 0 0 1/1 * ? *
+project.jobs.gui.groupExpandLevel=1
+project.later.executions.disable.value=0
+project.later.executions.disable=false
+project.later.executions.enable.value=
+project.later.executions.enable=false
+project.later.schedule.disable.value=
+project.later.schedule.disable=false
+project.later.schedule.enable.value=
+project.later.schedule.enable=false
+project.name=ansible-max-aliases
+project.nodeCache.enabled=false
+project.nodeCache.firstLoadSynch=true
+project.output.allowUnsanitized=false
+project.retry-counter=3
+project.ssh-authentication=privateKey
+resources.source.1.type=local
+resources.source.2.config.ansible-config-file-path=/home/rundeck/ansible-max-aliases/ansible.cfg
+resources.source.2.config.ansible-gather-facts=false
+resources.source.2.config.ansible-ignore-errors=true
+resources.source.2.config.ansible-inventory=/home/rundeck/ansible-max-aliases/inventory_max_aliases_52.ini
+resources.source.2.config.ansible-yaml-max-aliases=53
+resources.source.2.type=com.batix.rundeck.plugins.AnsibleResourceModelSourceFactory
+service.FileCopier.default.provider=sshj-scp
+service.NodeExecutor.default.provider=sshj-ssh

src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java

@@ -8,6 +8,7 @@
 
 import java.util.Arrays;
 import java.util.LinkedList;
+import java.util.Map;
 
 public interface AnsibleDescribable extends Describable {
 
@@ -90,6 +91,9 @@ public static String[] getValues() {
     	}
     }
 
+    // General variables
+    String SECONDARY = "SECONDARY";
+
     public static final String SERVICE_PROVIDER_TYPE = "ansible-service";
     public static final String ANSIBLE_PLAYBOOK_PATH = "ansible-playbook";
     public static final String ANSIBLE_PLAYBOOK_INLINE = "ansible-playbook-inline";
@@ -158,7 +162,14 @@ public static String[] getValues() {
 
     public static final String ANSIBLE_ENCRYPT_EXTRA_VARS = "ansible-encrypt-extra-vars";
 
-    String ANSIBLE_YAML_DATA_SIZE = "ansible-yaml-data-size";
+    // Inventory Yaml
+    String ANSIBLE_YAML_DATA_SIZE   = "ansible-yaml-data-size";
+    String ANSIBLE_YAML_MAX_ALIASES = "ansible-yaml-max-aliases";
+    String INVENTORY_YAML = "Inventory Yaml";
+    Map<String, Object> inventoryYamlOpt = Map.of(
+        StringRenderingConstants.GROUPING, SECONDARY,
+        StringRenderingConstants.GROUP_NAME, INVENTORY_YAML
+    );
 
     public static Property PLAYBOOK_PATH_PROP = PropertyUtil.string(
     			ANSIBLE_PLAYBOOK_PATH,
@@ -533,9 +544,20 @@ public static String[] getValues() {
     Property YAML_DATA_SIZE_PROP = PropertyBuilder.builder()
             .integer(ANSIBLE_YAML_DATA_SIZE)
             .required(false)
-            .title("Inventory Yaml Data Size")
-            .description("Set the MB size (Default value is 10)"+
-                    " therefore, the plugin can process the yaml data response coming from Ansible."+
+            .title("Data Size")
+            .description("Set the MB size (Default value is 10)."+
+                    " Allows the plugin to process the yaml data response coming from Ansible."+
+                    " (This only applies when Gather Facts = No)")
+            .renderingOptions(inventoryYamlOpt)
+            .build();
+
+    Property YAML_MAX_ALIASES_PROP = PropertyBuilder.builder()
+            .integer(ANSIBLE_YAML_MAX_ALIASES)
+            .required(false)
+            .title("Max Aliases")
+            .description("Set max size (Default value is 1000)."+
+                    " Allows to set the maximum number of aliases that the inventory can have."+
                     " (This only applies when Gather Facts = No)")
+            .renderingOptions(inventoryYamlOpt)
             .build();
 }

src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java

@@ -1,7 +1,6 @@
 package com.rundeck.plugins.ansible.plugin;
 
 import com.dtolabs.rundeck.core.common.Framework;
-import com.dtolabs.rundeck.core.common.INodeEntry;
 import com.dtolabs.rundeck.core.common.INodeSet;
 import com.dtolabs.rundeck.core.common.NodeEntryImpl;
 import com.dtolabs.rundeck.core.common.NodeSetImpl;
@@ -29,6 +28,7 @@
 import com.rundeck.plugins.ansible.util.VaultPrompt;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
 import org.rundeck.app.spi.Services;
 import org.rundeck.storage.api.PathUtil;
 import org.rundeck.storage.api.StorageException;
@@ -50,16 +50,16 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
-import java.util.stream.Collectors;
 
+import static com.rundeck.plugins.ansible.ansible.AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE;
+import static com.rundeck.plugins.ansible.ansible.AnsibleDescribable.ANSIBLE_YAML_MAX_ALIASES;
 import static com.rundeck.plugins.ansible.ansible.InventoryList.ALL;
 import static com.rundeck.plugins.ansible.ansible.InventoryList.CHILDREN;
 import static com.rundeck.plugins.ansible.ansible.InventoryList.HOSTS;
@@ -85,8 +85,6 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
 
   private String inventory;
   private boolean gatherFacts;
-  @Setter
-  private Integer yamlDataSize;
   private boolean ignoreErrors = false;
   private String limit;
   private String ignoreTagPrefix;
@@ -135,6 +133,11 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
 
   protected  String customTmpDirPath;
 
+  @Setter
+  private Integer yamlDataSize;
+  @Setter
+  private Integer yamlMaxAliases;
+
   @Setter
   private AnsibleInventoryList.AnsibleInventoryListBuilder ansibleInventoryListBuilder = null;
 
@@ -144,7 +147,7 @@ public AnsibleResourceModelSource(final Framework framework) {
       this.framework = framework;
   }
 
-    private static String resolveProperty(
+  private static String resolveProperty(
             final String attribute,
             final String defaultValue,
             final Properties configuration,
@@ -197,8 +200,6 @@ public void configure(Properties configuration) throws ConfigurationException {
     gatherFacts = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_GATHER_FACTS,null,configuration,executionDataContext));
     ignoreErrors = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_ERRORS,null,configuration,executionDataContext));
 
-    yamlDataSize = resolveIntProperty(AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext);
-
     limit = (String) resolveProperty(AnsibleDescribable.ANSIBLE_LIMIT,null,configuration,executionDataContext);
     ignoreTagPrefix = (String) resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_TAGS,null,configuration,executionDataContext);
 
@@ -254,6 +255,10 @@ public void configure(Properties configuration) throws ConfigurationException {
 
     encryptExtraVars = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_ENCRYPT_EXTRA_VARS,"false",configuration,executionDataContext));
 
+    // Inventory Yaml
+    yamlDataSize = resolveIntProperty(ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext);
+    yamlMaxAliases = resolveIntProperty(ANSIBLE_YAML_MAX_ALIASES,1000, configuration, executionDataContext);
+
   }
 
   public AnsibleRunner.AnsibleRunnerBuilder buildAnsibleRunner() throws ResourceModelSourceException {
@@ -708,10 +713,14 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB
     LoaderOptions snakeOptions = new LoaderOptions();
     // max inventory file size allowed to 10mb
     snakeOptions.setCodePointLimit(codePointLimit);
+    // max aliases. Default value is 1000
+    snakeOptions.setMaxAliasesForCollections(yamlMaxAliases);
     Yaml yaml = new Yaml(new SafeConstructor(snakeOptions));
 
     String listResp = getNodesFromInventory(runnerBuilder);
 
+    validateAliases(listResp);
+
     Map<String, Object> allInventory;
     try {
       allInventory = yaml.load(listResp);
@@ -971,4 +980,15 @@ private boolean isTagMapValid(Map<String, Object> tagMap, String tagName) {
     return true;
   }
 
+  /**
+   * Validates whether the YAML content contains aliases that exceed the maximum allowed.
+   * @param content String yaml
+   */
+  public void validateAliases(String content) {
+    int totalAliases = StringUtils.countMatches(content, ": *");
+    if (totalAliases > yamlMaxAliases) {
+      log.warn("The yaml inventory received has {} aliases and the maximum allowed is {}.", totalAliases, yamlMaxAliases);
+    }
+  }
+
 }

src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java

@@ -35,7 +35,6 @@ public AnsibleResourceModelSourceFactory(final Framework framework) {
         builder.property(INVENTORY_PROP);
         builder.property(CONFIG_FILE_PATH);
         builder.property(GATHER_FACTS_PROP);
-        builder.property(YAML_DATA_SIZE_PROP);
         builder.property(IGNORE_ERRORS_PROP);
         builder.property(LIMIT_PROP);
         builder.property(DISABLE_LIMIT_PROP);
@@ -64,6 +63,9 @@ public AnsibleResourceModelSourceFactory(final Framework framework) {
         builder.property(SSH_USE_AGENT);
         builder.property(BECOME_PASSWORD_STORAGE_PROP);
 
+        builder.property(YAML_DATA_SIZE_PROP);
+        builder.property(YAML_MAX_ALIASES_PROP);
+
         builder.mapping(ANSIBLE_INVENTORY,PROJ_PROP_PREFIX + ANSIBLE_INVENTORY);
         builder.frameworkMapping(ANSIBLE_INVENTORY,FWK_PROP_PREFIX + ANSIBLE_INVENTORY);
         builder.mapping(ANSIBLE_CONFIG_FILE_PATH,PROJ_PROP_PREFIX + ANSIBLE_CONFIG_FILE_PATH);