Skip to content

Commit 56ef33a

Browse files
ltamasterltamaster
authored
Merge pull request #382 from rundeck-plugins/enh/run-2670_ansible-model-source-yaml-parameter
RUN-2670: Enh: Added yaml data size parameter at model source
2 parents 188ad91 + 1c78865 commit 56ef33a

File tree

4 files changed

+141
-18
lines changed

4 files changed

+141
-18
lines changed

‎src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java

+11
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ public static String[] getValues() {
158158

159159
public static final String ANSIBLE_ENCRYPT_EXTRA_VARS = "ansible-encrypt-extra-vars";
160160

161+
String ANSIBLE_YAML_DATA_SIZE = "ansible-yaml-data-size";
162+
161163
public static Property PLAYBOOK_PATH_PROP = PropertyUtil.string(
162164
ANSIBLE_PLAYBOOK_PATH,
163165
"Playbook",
@@ -527,4 +529,13 @@ public static String[] getValues() {
527529
.title("Encrypt Extra Vars.")
528530
.description("Encrypt the value of the extra vars keys.")
529531
.build();
532+
533+
Property YAML_DATA_SIZE_PROP = PropertyBuilder.builder()
534+
.integer(ANSIBLE_YAML_DATA_SIZE)
535+
.required(false)
536+
.title("Inventory Yaml Data Size")
537+
.description("Set the MB size (Default value is 10)"+
538+
" therefore, the plugin can process the yaml data response coming from Ansible."+
539+
" (This only applies when Gather Facts = No)")
540+
.build();
530541
}

‎src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java

+40-12
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,14 @@
2525
import com.rundeck.plugins.ansible.ansible.AnsibleRunner;
2626
import com.rundeck.plugins.ansible.ansible.InventoryList;
2727
import com.rundeck.plugins.ansible.util.VaultPrompt;
28+
import lombok.Setter;
2829
import org.rundeck.app.spi.Services;
2930
import org.rundeck.storage.api.PathUtil;
3031
import org.rundeck.storage.api.StorageException;
3132
import org.yaml.snakeyaml.LoaderOptions;
3233
import org.yaml.snakeyaml.Yaml;
3334
import org.yaml.snakeyaml.constructor.SafeConstructor;
35+
import org.yaml.snakeyaml.error.YAMLException;
3436

3537
import java.io.BufferedReader;
3638
import java.io.ByteArrayOutputStream;
@@ -45,6 +47,7 @@
4547
import java.nio.file.SimpleFileVisitor;
4648
import java.nio.file.attribute.BasicFileAttributes;
4749
import java.util.ArrayList;
50+
import java.util.Collections;
4851
import java.util.HashMap;
4952
import java.util.HashSet;
5053
import java.util.List;
@@ -60,8 +63,10 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
6063
public static final String HOST_TPL_J2 = "host-tpl.j2";
6164
public static final String GATHER_HOSTS_YML = "gather-hosts.yml";
6265

66+
@Setter
6367
private Framework framework;
6468

69+
@Setter
6570
private Services services;
6671

6772
private String project;
@@ -72,6 +77,8 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
7277

7378
private String inventory;
7479
private boolean gatherFacts;
80+
@Setter
81+
private Integer yamlDataSize;
7582
private boolean ignoreErrors = false;
7683
private String limit;
7784
private String ignoreTagPrefix;
@@ -118,17 +125,14 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
118125

119126
protected boolean encryptExtraVars = false;
120127

128+
@Setter
121129
private AnsibleInventoryList.AnsibleInventoryListBuilder ansibleInventoryListBuilder = null;
122130

123131
public AnsibleResourceModelSource(final Framework framework) {
124132
this.framework = framework;
125133
}
126134

127-
public void setAnsibleInventoryListBuilder(AnsibleInventoryList.AnsibleInventoryListBuilder builder) {
128-
this.ansibleInventoryListBuilder = builder;
129-
}
130-
131-
private static String resolveProperty(
135+
private static String resolveProperty(
132136
final String attribute,
133137
final String defaultValue,
134138
final Properties configuration,
@@ -142,18 +146,32 @@ private static String resolveProperty(
142146
}
143147
}
144148

149+
private static Integer resolveIntProperty(
150+
final String attribute,
151+
final Integer defaultValue,
152+
final Properties configuration,
153+
final Map<String, Map<String, String>> dataContext) throws ConfigurationException {
154+
final String strValue = resolveProperty(attribute, null, configuration, dataContext);
155+
if (null != strValue) {
156+
try {
157+
return Integer.parseInt(strValue);
158+
} catch (NumberFormatException e) {
159+
throw new ConfigurationException("Can't parse attribute :" + attribute +
160+
", value: " + strValue +
161+
" Expected Integer. : " + e.getMessage(), e);
162+
}
163+
}
164+
return defaultValue;
165+
}
166+
145167
private static Boolean skipVar(final String hostVar, final List<String> varList) {
146168
for (final String specialVarString : varList) {
147169
if (hostVar.startsWith(specialVarString)) return true;
148170
}
149171
return false;
150172
}
151173

152-
public void setServices(Services services) {
153-
this.services = services;
154-
}
155-
156-
public void configure(Properties configuration) throws ConfigurationException {
174+
public void configure(Properties configuration) throws ConfigurationException {
157175

158176
project = configuration.getProperty("project");
159177
configDataContext = new HashMap<String, Map<String, String>>();
@@ -167,6 +185,8 @@ public void configure(Properties configuration) throws ConfigurationException {
167185
gatherFacts = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_GATHER_FACTS,null,configuration,executionDataContext));
168186
ignoreErrors = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_ERRORS,null,configuration,executionDataContext));
169187

188+
yamlDataSize = resolveIntProperty(AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext);
189+
170190
limit = (String) resolveProperty(AnsibleDescribable.ANSIBLE_LIMIT,null,configuration,executionDataContext);
171191
ignoreTagPrefix = (String) resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_TAGS,null,configuration,executionDataContext);
172192

@@ -670,14 +690,22 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
670690
*/
671691
public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerBuilder runnerBuilder) throws ResourceModelSourceException {
672692

693+
int codePointLimit = yamlDataSize * 1024 * 1024;
694+
673695
LoaderOptions snakeOptions = new LoaderOptions();
674696
// max inventory file size allowed to 10mb
675-
snakeOptions.setCodePointLimit(10_485_760);
697+
snakeOptions.setCodePointLimit(codePointLimit);
676698
Yaml yaml = new Yaml(new SafeConstructor(snakeOptions));
677699

678700
String listResp = getNodesFromInventory(runnerBuilder);
679701

680-
Map<String, Object> allInventory = yaml.load(listResp);
702+
Map<String, Object> allInventory;
703+
try {
704+
allInventory = yaml.load(listResp);
705+
} catch (YAMLException e) {
706+
throw new ResourceModelSourceException("Cannot load yaml data coming from Ansible: " + e.getMessage(), e);
707+
}
708+
681709
Map<String, Object> all = InventoryList.getValue(allInventory, ALL);
682710
Map<String, Object> children = InventoryList.getValue(all, CHILDREN);
683711

‎src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public AnsibleResourceModelSourceFactory(final Framework framework) {
3535
builder.property(INVENTORY_PROP);
3636
builder.property(CONFIG_FILE_PATH);
3737
builder.property(GATHER_FACTS_PROP);
38+
builder.property(YAML_DATA_SIZE_PROP);
3839
builder.property(IGNORE_ERRORS_PROP);
3940
builder.property(LIMIT_PROP);
4041
builder.property(DISABLE_LIMIT_PROP);

‎src/test/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceSpec.groovy

+89-6
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
package com.rundeck.plugins.ansible.plugin
22

3-
import com.dtolabs.rundeck.core.storage.StorageTree
4-
import com.dtolabs.rundeck.core.storage.keys.KeyStorageTree
5-
import org.rundeck.app.spi.Services
6-
7-
import static com.rundeck.plugins.ansible.ansible.AnsibleInventoryList.AnsibleInventoryListBuilder
8-
93
import com.dtolabs.rundeck.core.common.Framework
104
import com.dtolabs.rundeck.core.common.INodeEntry
115
import com.dtolabs.rundeck.core.common.INodeSet
126
import com.dtolabs.rundeck.core.resources.ResourceModelSource
7+
import com.dtolabs.rundeck.core.resources.ResourceModelSourceException
8+
import com.dtolabs.rundeck.core.storage.keys.KeyStorageTree
139
import com.rundeck.plugins.ansible.ansible.AnsibleDescribable
1410
import com.rundeck.plugins.ansible.ansible.AnsibleInventoryList
11+
import org.rundeck.app.spi.Services
1512
import org.yaml.snakeyaml.Yaml
13+
import org.yaml.snakeyaml.error.YAMLException
1614
import spock.lang.Specification
1715

16+
import static com.rundeck.plugins.ansible.ansible.AnsibleInventoryList.AnsibleInventoryListBuilder
17+
1818
/**
1919
* AnsibleResourceModelSource test
2020
*/
@@ -94,4 +94,87 @@ class AnsibleResourceModelSourceSpec extends Specification {
9494
'NODE_3' | 'ansible_os_family' | 'ansible_os_name' | 'ansible_kernel' | 'ansible_architecture' | 'ansible_user_id' | 'ansible_distribution'
9595
}
9696

97+
void "ansible yaml data size parameter without an Exception"() {
98+
given:
99+
Framework framework = Mock(Framework) {
100+
getBaseDir() >> Mock(File) {
101+
getAbsolutePath() >> '/tmp'
102+
}
103+
}
104+
ResourceModelSource plugin = new AnsibleResourceModelSource(framework)
105+
Properties config = new Properties()
106+
config.put('project', 'project_1')
107+
config.put(AnsibleDescribable.ANSIBLE_GATHER_FACTS, 'false')
108+
plugin.configure(config)
109+
Services services = Mock(Services) {
110+
getService(KeyStorageTree.class) >> Mock(KeyStorageTree)
111+
}
112+
plugin.setServices(services)
113+
plugin.yamlDataSize = 2
114+
115+
when: "small inventory"
116+
AnsibleInventoryListBuilder inventoryListBuilder = mockInventoryList(qtyNodes)
117+
plugin.ansibleInventoryListBuilder = inventoryListBuilder
118+
INodeSet nodes = plugin.getNodes()
119+
120+
then: "non exception is thrown because data can be handled"
121+
notThrown(YAMLException)
122+
nodes.size() == qtyNodes
123+
124+
where:
125+
qtyNodes | _
126+
2_0000 | _
127+
3_0000 | _
128+
}
129+
130+
void "ansible yaml data size parameter with an Exception"() {
131+
given:
132+
Framework framework = Mock(Framework) {
133+
getBaseDir() >> Mock(File) {
134+
getAbsolutePath() >> '/tmp'
135+
}
136+
}
137+
ResourceModelSource plugin = new AnsibleResourceModelSource(framework)
138+
Properties config = new Properties()
139+
config.put('project', 'project_1')
140+
config.put(AnsibleDescribable.ANSIBLE_GATHER_FACTS, 'false')
141+
plugin.configure(config)
142+
Services services = Mock(Services) {
143+
getService(KeyStorageTree.class) >> Mock(KeyStorageTree)
144+
}
145+
plugin.setServices(services)
146+
plugin.yamlDataSize = 2
147+
148+
when: "huge inventory"
149+
AnsibleInventoryListBuilder inventoryListBuilder = mockInventoryList(100_000)
150+
plugin.ansibleInventoryListBuilder = inventoryListBuilder
151+
plugin.getNodes()
152+
153+
then: "throws an exception because data is too big to be precessed"
154+
thrown(ResourceModelSourceException)
155+
}
156+
157+
private AnsibleInventoryListBuilder mockInventoryList(int qtyNodes) {
158+
return Mock(AnsibleInventoryListBuilder) {
159+
build() >> Mock(AnsibleInventoryList) {
160+
getNodeList() >> createNodes(qtyNodes)
161+
}
162+
}
163+
}
164+
165+
private static String createNodes(int qty) {
166+
Yaml yaml = new Yaml()
167+
Map<String, Object> host = [:]
168+
for (int i=1; i <= qty; i++) {
169+
String nodeName = "node-$i"
170+
String hostValue = "any-name-$i"
171+
host.put((nodeName), ['hostname' : (hostValue)])
172+
}
173+
def hosts = ['hosts' : host]
174+
def groups = ['ungrouped' : hosts]
175+
def children = ['children' : groups]
176+
def all = ['all' : children]
177+
return yaml.dump(all)
178+
}
179+
97180
}

0 commit comments

Comments
 (0)