issue-131 - Add pagination to integration cleanups

.editorconfig

@@ -0,0 +1,18 @@
+root = true
+
+[*]
+indent_style = space
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf
+
+[*.json]
+indent_size = 4
+indent_style = space
+
+[*.py]
+indent_style = space
+
+[*.{yaml,yml}]
+indent_style = space

.gitignore

@@ -2,4 +2,5 @@ cloud-config-vultr.ini
 output
 resources
 vultr-cloud-*.tar.gz
-.vscode/
+.ansible/
+.vscode/

tests/integration/targets/cleanup/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
 vultr_api_url: https://api.vultr.com/v2
 vultr_api_key: "{{ lookup('env', 'VULTR_API_KEY') }}"
+vultr_api_results_per_page: 100

tests/integration/targets/cleanup/tasks/cleanup_bare_metal.yml

@@ -4,16 +4,31 @@
   block:
     - name: List bare metals
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/bare-metals"
+        url: "{{ vultr_api_url }}/bare-metals?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/bare-metals?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        bare_metals: "{{ initial_res.json.bare_metals + (res.results | map(attribute='json') | map(attribute='bare_metals') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: bare_metals
 
     - name: Remove all bare metals created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.label
-      with_items: "{{ res.json.bare_metals }}"
+      with_items: "{{ bare_metals }}"
       loop_control:
         label: "{{ item.label }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_block_storage.yml

@@ -4,16 +4,31 @@
   block:
     - name: List blocks
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/blocks"
+        url: "{{ vultr_api_url }}/blocks?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/blocks?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        blocks: "{{ initial_res.json.blocks + (res.results | map(attribute='json') | map(attribute='blocks') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: blocks
 
     - name: Remove all blocks created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.label
-      with_items: "{{ res.json.blocks }}"
+      with_items: "{{ blocks }}"
       loop_control:
         label: "{{ item.label }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_dns_domain.yml

@@ -4,16 +4,31 @@
   block:
     - name: List dns domains
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/domains"
+        url: "{{ vultr_api_url }}/domains?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/domains?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        domains: "{{ initial_res.json.domains + (res.results | map(attribute='json') | map(attribute='domains') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: domains
 
     - name: Remove all dns domains created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.domain
-      with_items: "{{ res.json.domains }}"
+      with_items: "{{ domains }}"
       loop_control:
         label: "{{ item.domain }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_firewall_group.yml

@@ -4,16 +4,31 @@
   block:
     - name: List firewall groups
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/firewalls"
+        url: "{{ vultr_api_url }}/firewalls?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/firewalls?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        firewall_groups: "{{ initial_res.json.firewall_groups + (res.results | map(attribute='json') | map(attribute='firewall_groups') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: firewall_groups
 
     - name: Remove all firewall groups created by this test run
       ansible.builtin.uri:
@@ -23,6 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.description
-      with_items: "{{ res.json.firewall_groups }}"
+      with_items: "{{ firewall_groups }}"
       loop_control:
         label: "{{ item.description }}"
+      no_log: true

tests/integration/targets/cleanup/tasks/cleanup_instance.yml

@@ -4,16 +4,31 @@
   block:
     - name: List instances
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/instances"
+        url: "{{ vultr_api_url }}/instances?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/instances?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        instances: "{{ initial_res.json.instances + (res.results | map(attribute='json') | map(attribute='instances') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: instances
 
     - name: Remove all instances created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.label
-      with_items: "{{ res.json.instances }}"
+      with_items: "{{ instances }}"
       loop_control:
         label: "{{ item.label }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_network.yml

@@ -4,16 +4,31 @@
   block:
     - name: List networks
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/private-networks"
+        url: "{{ vultr_api_url }}/private-networks?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/private-networks?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        networks: "{{ initial_res.json.networks + (res.results | map(attribute='json') | map(attribute='networks') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: networks
 
     - name: Remove all networks created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.description
-      with_items: "{{ res.json.networks }}"
+      with_items: "{{ networks }}"
       loop_control:
         label: "{{ item.description }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_object_storage.yml

@@ -4,16 +4,31 @@
   block:
     - name: List object storages
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/object-storage"
+        url: "{{ vultr_api_url }}/object-storage?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/object-storage?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        object_storages: "{{ initial_res.json.object_storages + (res.results | map(attribute='json') | map(attribute='object_storages') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: object_storages
 
     - name: Remove all object storages created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.label
-      with_items: "{{ res.json.object_storages }}"
+      with_items: "{{ object_storages }}"
       loop_control:
         label: "{{ item.label }}"
       no_log: true

tests/integration/targets/cleanup/tasks/cleanup_reserved_ip.yml

@@ -4,16 +4,31 @@
   block:
     - name: List reserved-ips
       ansible.builtin.uri:
-        url: "{{ vultr_api_url }}/reserved-ips"
+        url: "{{ vultr_api_url }}/reserved-ips?per_page={{ vultr_api_results_per_page }}"
+        headers:
+          Authorization: Bearer {{ vultr_api_key }}
+        status_code: 200
+      register: initial_res
+      no_log: true
+
+    - name: Page through subsequent results
+      ansible.builtin.uri:
+        url: "{{ vultr_api_url }}/reserved-ips?per_page={{ vultr_api_results_per_page }}&cursor={{ res.json.meta.links.next | default(initial_res.json.meta.links.next) }}"
         headers:
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 200
       register: res
+      loop: "{{ range(1, ((initial_res.json.meta.total | int) / (vultr_api_results_per_page | int)) | round(0, 'ceil') | int) | list }}"
+      when: initial_res.json.meta.links.next != ''
       no_log: true
 
+    - name: Compile results
+      ansible.builtin.set_fact:
+        reserved_ips: "{{ initial_res.json.reserved_ips + (res.results | map(attribute='json') | map(attribute='reserved_ips') | flatten) }}"
+
     - name: Found resources
       ansible.builtin.debug:
-        var: res.json
+        var: reserved_ips
 
     - name: Remove all reserved-ips created by this test run
       ansible.builtin.uri:
@@ -23,7 +38,7 @@
           Authorization: Bearer {{ vultr_api_key }}
         status_code: 204
       when: vultr_resource_prefix in item.label
-      with_items: "{{ res.json.reserved_ips }}"
+      with_items: "{{ reserved_ips }}"
       loop_control:
         label: "{{ item.label }}"
       no_log: true