From ce08e41c8595e6368eb28b98bc0dbdc8ce4bead9 Mon Sep 17 00:00:00 2001 From: Jeff Sites Date: Thu, 8 Dec 2022 07:49:59 -0500 Subject: [PATCH 1/3] WIP: vultr: inventory plugin and unit tests --- .github/workflows/integration.yml | 2 +- .github/workflows/unit.yml | 76 +++++ .gitignore | 1 + galaxy.yml | 3 +- plugins/inventory/vultr.py | 286 ++++++++++++++++++ plugins/module_utils/vultr_v2.py | 6 +- ...origin => cloud-config-vultr.ini.template} | 0 .../fixtures/empty_vultr_inventory.json | 10 + .../unauthorized_vultr_inventory.json | 4 + .../inventory/fixtures/vultr_inventory.json | 275 +++++++++++++++++ .../fixtures/vultr_inventory_page1.json | 179 +++++++++++ .../fixtures/vultr_inventory_page2.json | 116 +++++++ tests/unit/plugins/inventory/test_vultr.py | 233 ++++++++++++++ 13 files changed, 1188 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/unit.yml create mode 100644 plugins/inventory/vultr.py rename tests/integration/{cloud-config-vultr.ini.origin => cloud-config-vultr.ini.template} (100%) create mode 100644 tests/unit/plugins/inventory/fixtures/empty_vultr_inventory.json create mode 100644 tests/unit/plugins/inventory/fixtures/unauthorized_vultr_inventory.json create mode 100644 tests/unit/plugins/inventory/fixtures/vultr_inventory.json create mode 100644 tests/unit/plugins/inventory/fixtures/vultr_inventory_page1.json create mode 100644 tests/unit/plugins/inventory/fixtures/vultr_inventory_page2.json create mode 100644 tests/unit/plugins/inventory/test_vultr.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4a318a4..5be421f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ on: paths: - ".github/workflows/integration.yml" - "plugins/**" - - "tests/**" + - "tests/integration/**" schedule: - cron: 56 3 * * 2 # Run weekly diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml new file mode 100644 index 0000000..4b3cbd4 --- /dev/null +++ b/.github/workflows/unit.yml @@ -0,0 +1,76 @@ +--- +name: Unit tests + +on: + pull_request: + types: [labeled] + push: + branches: + - main + paths: + - '.github/workflows/unit.yml' + - 'plugins/**' + - 'tests/unit/**' + +jobs: + unit-test: + name: Unit test Ansible ${{ matrix.ansible }} Py${{ matrix.python }} + if: ${{ github.event.label.name == 'automation' || github.ref_name == 'main' }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ansible_collections/vultr/cloud + strategy: + fail-fast: false + matrix: + ansible: + - stable-2.11 + - stable-2.13 + python: + - 2.7 + - 3.6 + - 3.8 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + path: ansible_collections/vultr/cloud + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install ansible-base (${{ matrix.ansible }}) + run: | + python -m pip install --upgrade pip + pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check + + - name: Build and install collection + run: | + ansible-galaxy collection build . + ansible-galaxy collection install *.gz + + - name: Run the tests + run: >- + ansible-test + units + --docker + -v + --color + --continue-on-error + --diff + --python ${{ matrix.python }} + --coverage + + - name: Generate coverage report + run: >- + ansible-test + coverage xml + -v + --requirements + --group-by command + --group-by version + - uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index ace109b..6c70190 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cloud-config-vultr.ini output resources +vultr-cloud-*.tar.gz diff --git a/galaxy.yml b/galaxy.yml index 940b1e6..c43b060 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,6 +2,7 @@ authors: - "René Moser (@resmo)" - "Yanis Guenane (@Spredzy)" + - "jasites (@jasites)" dependencies: {} description: "Ansible Collection for Vultr Cloud" documentation: "https://docs.ansible.com/ansible/latest/collections/vultr/cloud/" @@ -15,4 +16,4 @@ repository: "https://github.com/vultr/ansible-collection-vultr" tags: - cloud - vultr -version: 1.3.1 +version: 1.4.0 diff --git a/plugins/inventory/vultr.py b/plugins/inventory/vultr.py new file mode 100644 index 0000000..c9dbd10 --- /dev/null +++ b/plugins/inventory/vultr.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) jasites +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +# flake8: noqa: E402 + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +DOCUMENTATION = """ +--- +name: vultr +short_description: Retrieves list of instances via Vultr v2 API +description: + - Vultr inventory plugin. + - Retrieves list of instances via Vultr v2 API. + - Configuration of this plugin is done in '(vultr|vultr_hosts|vultr_instances).(yaml|yml)' +version_added: '1.4.0' +author: + - jasites (@jasites) +extends_documentation_fragment: + - constructed + - inventory_cache +options: + api_endpoint: + description: + - URL to API endpint (without trailing slash). + - Fallback environment variable C(VULTR_API_ENDPOINT). + type: str + env: + - name: VULTR_API_ENDPOINT + default: https://api.vultr.com/v2 + api_key: + description: + - API key of the Vultr API. + - Fallback environment variable C(VULTR_API_KEY). + type: str + env: + - name: VULTR_API_KEY + required: true + api_results_per_page: + description: + - When receiving large numbers of instances, specify how many instances should be returned per call to API. + - This does not determine how many results are returned; all instances are returned according to other filters. + - Vultr API maximum is 500. + - Fallback environment variable C(VULTR_API_RESULTS_PER_PAGE) + type: int + env: + - name: VULTR_API_RESULTS_PER_PAGE + default: 100 + api_timeout: + description: + - HTTP timeout to Vultr API. + - Fallback environment variable C(VULTR_API_TIMEOUT). + type: int + env: + - name: VULTR_API_TIMEOUT + default: 60 + attributes: + description: + - Instance attributes to add as host variables to each host added to inventory. + - See U(https://www.vultr.com/api/#operation/list-instances) for valid values. + type: list + elements: str + default: + - id + - region + - label + - plan + - hostname + - main_ip + filters: + description: + - Filter hosts with Jinja2 templates. + - If not provided, all hosts are added to inventory. + type: list + elements: str + default: [] + plugin: + description: + - Name of Vultr inventory plugin. + - This should always be C(vultr.cloud.vultr). + type: str + choices: ['vultr.cloud.vultr'] + required: true + variable_prefix: + description: + - Prefix of generated variables (e.g. C(id) becomes C(vultr_id)) + type: str + default: 'vultr_' + validate_certs: + description: + - Validate SSL certs of the Vultr API. + type: bool + default: true +notes: + - Also see the API documentation on U(https://www.vultr.com/api/). +""" + +EXAMPLES = """ +--- +# vultr{,-{hosts,instances}}.y{,a}ml +# All configuration done via environment variables: +plugin: vultr.cloud.vultr + +# Grouping and filtering configuration in inventory file +plugin: vultr.cloud.vultr +api_key: '{{ lookup("pipe"), "./get_vultr_api_key.sh" }}' +keyed_groups: + - key: vultr_tags | lower + prefix: '' + separator: '' +filters: + - '"vpc" in vultr_tags' + - 'vultr_plan == "vc2-2c-4gb"' +""" + +RETURN = r""" # """ + +import json +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils._text import to_native +from ansible.module_utils.urls import Request +from ansible.module_utils.six.moves.urllib.error import URLError, HTTPError +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable + +from ..module_utils.vultr_v2 import VULTR_USER_AGENT + + +class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): + + NAME = "vultr.cloud.vultr" + + def _get_instances(self): + instances = [] + api_key = self.get_option("api_key") + if self.templar.is_template(api_key): + api_key = self.templar.template(api_key) + + headers = { + "Content-Type": "application/json", + "User-Agent": VULTR_USER_AGENT, + "Authorization": "Bearer {0}".format(api_key), + } + + self.req = Request( + headers=headers, + timeout=int(self.get_option("api_timeout")), # type: ignore + validate_certs=self.get_option("validate_certs"), # type: ignore + ) + + api_endpoint = "{0}/instances?per_page={1}".format( + self.get_option("api_endpoint"), self.get_option("api_results_per_page") + ) + + cursor = "" + req_url = api_endpoint + try: + while True: + self.display.vvv("Querying API: {0}".format(req_url)) + + page = json.load(self.req.get(req_url)) + instances.extend(page["instances"]) + cursor = page["meta"]["links"]["next"] + + if cursor == "": + return instances + + req_url = "{0}&cursor={1}".format(api_endpoint, cursor) + + except (KeyError, ValueError): + raise AnsibleParserError("Unable to parse JSON response.") + except (URLError, HTTPError) as err: + raise AnsibleParserError(err) + + def _populate(self, instances): + attributes = self.get_option("attributes") + host_filters = self.get_option("filters") + strict = self.get_option("strict") + variable_prefix = self.get_option("variable_prefix") + + for instance in instances: + instance_label = instance.get("label") + + if not instance_label: + continue + + host_variables = {} + for k, v in instance.items(): + if k in attributes: + host_variables["{0}{1}".format(variable_prefix, k)] = v + + if not self._passes_filters( + host_filters, host_variables, instance_label, strict # type: ignore + ): + self.display.vvv("Host {0} excluded by filters".format(instance_label)) + continue + + self.inventory.add_host(instance_label) + + for var_name, var_val in host_variables.items(): + self.inventory.set_variable(instance_label, var_name, var_val) + + self._set_composite_vars( + self.get_option("compose"), + self.inventory.get_host(instance_label).get_vars(), + instance_label, + strict, # type: ignore + ) + + self._add_host_to_composed_groups( + self.get_option("groups"), dict(), instance_label, strict # type: ignore + ) + + self._add_host_to_keyed_groups( + self.get_option("keyed_groups"), dict(), instance_label, strict # type: ignore + ) + + def _passes_filters(self, filters, variables, host, strict=False): + if filters and isinstance(filters, list): + for template in filters: + try: + if not self._compose(template, variables): + return False + except Exception as e: + if strict: + raise AnsibleError( + "Could not evaluate host filter {0} for {1}: {2}".format( + template, host, to_native(e) + ) + ) + return False + return True + + def verify_file(self, path): + valid = False + if super(InventoryModule, self).verify_file(path): + if path.endswith( + ( + "vultr.yaml", + "vultr.yml", + "vultr_hosts.yaml", + "vultr_hosts.yml", + "vultr_instances.yaml", + "vultr_instances.yml", + ) + ): + valid = True + else: + self.display.vvv( + "Skipping due to inventory configuration file name mismatch. " + "Valid filenames: " + "vultr.yaml, vultr.yml, vultr_hosts.yaml, vultr_hosts.yml, " + "vultr_instances.yaml, vultr_instances.yml" + ) + return valid + + def parse(self, inventory, loader, path, cache=True): + super(InventoryModule, self).parse(inventory, loader, path) + + self._read_config_data(path) + + cache_key = self.get_cache_key(path) + use_cache = self.get_option("cache") and cache + update_cache = self.get_option("cache") and not cache + + instances = None + if use_cache: + try: + instances = self._cache[cache_key] + except KeyError: + update_cache = True + + if instances is None: + instances = self._get_instances() + + if update_cache: + self._cache[cache_key] = instances + + self._populate(instances) diff --git a/plugins/module_utils/vultr_v2.py b/plugins/module_utils/vultr_v2.py index c381ee5..55f9761 100644 --- a/plugins/module_utils/vultr_v2.py +++ b/plugins/module_utils/vultr_v2.py @@ -35,7 +35,11 @@ def vultr_argument_spec(): fallback=(env_fallback, ["VULTR_API_TIMEOUT"]), default=60, ), - api_retries=dict(type="int", fallback=(env_fallback, ["VULTR_API_RETRIES"]), default=5), + api_retries=dict( + type="int", + fallback=(env_fallback, ["VULTR_API_RETRIES"]), + default=5 + ), api_retry_max_delay=dict( type="int", fallback=(env_fallback, ["VULTR_API_RETRY_MAX_DELAY"]), diff --git a/tests/integration/cloud-config-vultr.ini.origin b/tests/integration/cloud-config-vultr.ini.template similarity index 100% rename from tests/integration/cloud-config-vultr.ini.origin rename to tests/integration/cloud-config-vultr.ini.template diff --git a/tests/unit/plugins/inventory/fixtures/empty_vultr_inventory.json b/tests/unit/plugins/inventory/fixtures/empty_vultr_inventory.json new file mode 100644 index 0000000..4ae5859 --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/empty_vultr_inventory.json @@ -0,0 +1,10 @@ +{ + "instances": [], + "meta": { + "total": 0, + "links": { + "next": "", + "prev": "" + } + } +} diff --git a/tests/unit/plugins/inventory/fixtures/unauthorized_vultr_inventory.json b/tests/unit/plugins/inventory/fixtures/unauthorized_vultr_inventory.json new file mode 100644 index 0000000..8da6d13 --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/unauthorized_vultr_inventory.json @@ -0,0 +1,4 @@ +{ + "error": "Invalid API token.", + "status": 401 +} diff --git a/tests/unit/plugins/inventory/fixtures/vultr_inventory.json b/tests/unit/plugins/inventory/fixtures/vultr_inventory.json new file mode 100644 index 0000000..11ae1c7 --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/vultr_inventory.json @@ -0,0 +1,275 @@ +[ + { + "id": "f1c4ae0c-ea06-4449-a028-ffe6b05f9f80", + "os": "Custom Installed", + "ram": 512, + "disk": 10, + "main_ip": "100.68.102.38", + "vcpu_count": 1, + "region": "ewr", + "plan": "vc2-1c-0.5gb-v6", + "date_created": "2022-12-05T13:15:58+00:00", + "status": "active", + "allowed_bandwidth": 500, + "netmask_v4": "255.255.192.0", + "gateway_v4": "100.68.64.1", + "power_status": "running", + "server_status": "locked", + "v6_network": "2001:19f0:face:395c::", + "v6_main_ip": "2001:19f0:face:395c:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "", + "internal_ip": "10.1.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6", + "ddos_protection" + ] + }, + { + "id": "4adbf858-8426-40c4-8f48-91d83024bb5e", + "os": "Custom Installed", + "ram": 1024, + "disk": 25, + "main_ip": "100.68.102.39", + "vcpu_count": 1, + "region": "sea", + "plan": "vc2-1c-1gb", + "date_created": "2022-12-05T13:21:57+00:00", + "status": "active", + "allowed_bandwidth": 1000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "104.207.156.1", + "power_status": "stopped", + "server_status": "locked", + "v6_network": "2001:19f0:face:f17::", + "v6_main_ip": "2001:19f0:face:0f17:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "restore-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "restore-guest", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "ipv6" + ] + }, + { + "id": "5d8813ec-df65-4a69-8bfc-864aabe2a7d2", + "os": "Custom Installed", + "ram": 1024, + "disk": 25, + "main_ip": "100.68.102.40", + "vcpu_count": 1, + "region": "ord", + "plan": "vhp-1c-1gb-intel", + "date_created": "2022-12-05T13:29:19+00:00", + "status": "pending", + "allowed_bandwidth": 2000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.76.228.1", + "power_status": "running", + "server_status": "none", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "snapshot-guest", + "internal_ip": "", + "kvm": "", + "hostname": "snapshot-guest", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + }, + { + "id": "30c17c34-223f-436e-acbe-393af5293adb", + "os": "Debian 11 x64 (bullseye)", + "ram": 512, + "disk": 10, + "main_ip": "100.68.102.41", + "vcpu_count": 1, + "region": "ewr", + "plan": "vc2-1c-0.5gb-v6", + "date_created": "2022-12-05T12:43:00+00:00", + "status": "active", + "allowed_bandwidth": 500, + "netmask_v4": "255.255.192.0", + "gateway_v4": "100.68.64.1", + "power_status": "running", + "server_status": "ok", + "v6_network": "2001:19f0:face:8fd::", + "v6_main_ip": "2001:19f0:face:08fd:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "debian-guest", + "internal_ip": "10.1.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "debian-guest", + "tag": "", + "tags": [ + "vpc", + "vpc:d0db548b-5d50-4c94-9cdb-b083d46925f8", + "os:debian" + ], + "os_id": 477, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + }, + { + "id": "37d67f9f-17f0-4c56-879b-507355dc5174", + "os": "OpenBSD 7.2 x64", + "ram": 4096, + "disk": 30, + "main_ip": "100.68.102.42", + "vcpu_count": 1, + "region": "ewr", + "plan": "voc-g-1c-4gb-30s-amd", + "date_created": "2022-12-05T12:44:17+00:00", + "status": "active", + "allowed_bandwidth": 4000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.63.22.1", + "power_status": "running", + "server_status": "ok", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "openbsd-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "openbsd-guest", + "tag": "", + "tags": [], + "os_id": 1968, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + }, + { + "id": "2db0bb8c-9d83-4e62-86ef-b3d999960d72", + "os": "Windows 2016 Standard", + "ram": 2048, + "disk": 50, + "main_ip": "100.68.102.43", + "vcpu_count": 1, + "region": "fra", + "plan": "vhp-1c-2gb-amd", + "date_created": "2022-12-05T12:50:30+00:00", + "status": "active", + "allowed_bandwidth": 3000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.77.142.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "windows-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "windows-guest", + "tag": "", + "tags": [ + "os:windows2016" + ], + "os_id": 240, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + }, + { + "id": "e0aabdd7-7d68-448a-a0ce-a7712b09fcc1", + "os": "Arch Linux x64", + "ram": 6144, + "disk": 70, + "main_ip": "100.68.102.44", + "vcpu_count": 1, + "region": "sjc", + "plan": "vcg-a100-1c-6g-4vram", + "date_created": "2022-12-05T12:52:06+00:00", + "status": "active", + "allowed_bandwidth": 1000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "140.82.50.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "2001:19f0:face:1df3::", + "v6_main_ip": "2001:19f0:face:1df3:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "mldev-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "mldev-guest", + "tag": "", + "tags": [], + "os_id": 535, + "app_id": 0, + "image_id": "mldev", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + }, + { + "id": "90676f7b-5a34-49a8-b770-b0abed017577", + "os": "Application", + "ram": 2048, + "disk": 64, + "main_ip": "100.68.102.45", + "vcpu_count": 1, + "region": "ord", + "plan": "vhf-1c-2gb", + "date_created": "2022-12-05T12:54:37+00:00", + "status": "active", + "allowed_bandwidth": 2000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "149.28.114.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "2001:19f0:face:16ec::", + "v6_main_ip": "2001:19f0:face:16ec:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "wordpress-guest", + "internal_ip": "10.2.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "wordpress-guest", + "tag": "", + "tags": [ + "vpc", + "vpc:d0db548b-5d50-4c94-9cdb-b083d46925f8", + "application" + ], + "os_id": 186, + "app_id": 2, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + } +] diff --git a/tests/unit/plugins/inventory/fixtures/vultr_inventory_page1.json b/tests/unit/plugins/inventory/fixtures/vultr_inventory_page1.json new file mode 100644 index 0000000..a486228 --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/vultr_inventory_page1.json @@ -0,0 +1,179 @@ +{ + "instances": [ + { + "id": "f1c4ae0c-ea06-4449-a028-ffe6b05f9f80", + "os": "Custom Installed", + "ram": 512, + "disk": 10, + "main_ip": "100.68.102.38", + "vcpu_count": 1, + "region": "ewr", + "plan": "vc2-1c-0.5gb-v6", + "date_created": "2022-12-05T13:15:58+00:00", + "status": "active", + "allowed_bandwidth": 500, + "netmask_v4": "255.255.192.0", + "gateway_v4": "100.68.64.1", + "power_status": "running", + "server_status": "locked", + "v6_network": "2001:19f0:face:395c::", + "v6_main_ip": "2001:19f0:face:395c:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "", + "internal_ip": "10.1.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6", + "ddos_protection" + ] + }, + { + "id": "4adbf858-8426-40c4-8f48-91d83024bb5e", + "os": "Custom Installed", + "ram": 1024, + "disk": 25, + "main_ip": "100.68.102.39", + "vcpu_count": 1, + "region": "sea", + "plan": "vc2-1c-1gb", + "date_created": "2022-12-05T13:21:57+00:00", + "status": "active", + "allowed_bandwidth": 1000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "104.207.156.1", + "power_status": "stopped", + "server_status": "locked", + "v6_network": "2001:19f0:face:f17::", + "v6_main_ip": "2001:19f0:face:0f17:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "restore-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "restore-guest", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "ipv6" + ] + }, + { + "id": "5d8813ec-df65-4a69-8bfc-864aabe2a7d2", + "os": "Custom Installed", + "ram": 1024, + "disk": 25, + "main_ip": "100.68.102.40", + "vcpu_count": 1, + "region": "ord", + "plan": "vhp-1c-1gb-intel", + "date_created": "2022-12-05T13:29:19+00:00", + "status": "pending", + "allowed_bandwidth": 2000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.76.228.1", + "power_status": "running", + "server_status": "none", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "snapshot-guest", + "internal_ip": "", + "kvm": "", + "hostname": "snapshot-guest", + "tag": "", + "tags": [], + "os_id": 159, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + }, + { + "id": "30c17c34-223f-436e-acbe-393af5293adb", + "os": "Debian 11 x64 (bullseye)", + "ram": 512, + "disk": 10, + "main_ip": "100.68.102.41", + "vcpu_count": 1, + "region": "ewr", + "plan": "vc2-1c-0.5gb-v6", + "date_created": "2022-12-05T12:43:00+00:00", + "status": "active", + "allowed_bandwidth": 500, + "netmask_v4": "255.255.192.0", + "gateway_v4": "100.68.64.1", + "power_status": "running", + "server_status": "ok", + "v6_network": "2001:19f0:face:8fd::", + "v6_main_ip": "2001:19f0:face:08fd:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "debian-guest", + "internal_ip": "10.1.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "debian-guest", + "tag": "", + "tags": [ + "vpc", + "vpc:d0db548b-5d50-4c94-9cdb-b083d46925f8", + "os:debian" + ], + "os_id": 477, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + }, + { + "id": "37d67f9f-17f0-4c56-879b-507355dc5174", + "os": "OpenBSD 7.2 x64", + "ram": 4096, + "disk": 30, + "main_ip": "100.68.102.42", + "vcpu_count": 1, + "region": "ewr", + "plan": "voc-g-1c-4gb-30s-amd", + "date_created": "2022-12-05T12:44:17+00:00", + "status": "active", + "allowed_bandwidth": 4000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.63.22.1", + "power_status": "running", + "server_status": "ok", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "openbsd-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "openbsd-guest", + "tag": "", + "tags": [], + "os_id": 1968, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + } + ], + "meta": { + "total": 8, + "links": { + "next": "page2", + "prev": "" + } + } +} diff --git a/tests/unit/plugins/inventory/fixtures/vultr_inventory_page2.json b/tests/unit/plugins/inventory/fixtures/vultr_inventory_page2.json new file mode 100644 index 0000000..2f4ace8 --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/vultr_inventory_page2.json @@ -0,0 +1,116 @@ +{ + "instances": [ + { + "id": "2db0bb8c-9d83-4e62-86ef-b3d999960d72", + "os": "Windows 2016 Standard", + "ram": 2048, + "disk": 50, + "main_ip": "100.68.102.43", + "vcpu_count": 1, + "region": "fra", + "plan": "vhp-1c-2gb-amd", + "date_created": "2022-12-05T12:50:30+00:00", + "status": "active", + "allowed_bandwidth": 3000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "45.77.142.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "", + "v6_main_ip": "", + "v6_network_size": 0, + "label": "windows-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "windows-guest", + "tag": "", + "tags": [ + "os:windows2016" + ], + "os_id": 240, + "app_id": 0, + "image_id": "", + "firewall_group_id": "", + "features": [] + }, + { + "id": "e0aabdd7-7d68-448a-a0ce-a7712b09fcc1", + "os": "Arch Linux x64", + "ram": 6144, + "disk": 70, + "main_ip": "100.68.102.44", + "vcpu_count": 1, + "region": "sjc", + "plan": "vcg-a100-1c-6g-4vram", + "date_created": "2022-12-05T12:52:06+00:00", + "status": "active", + "allowed_bandwidth": 1000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "140.82.50.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "2001:19f0:face:1df3::", + "v6_main_ip": "2001:19f0:face:1df3:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "mldev-guest", + "internal_ip": "", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "mldev-guest", + "tag": "", + "tags": [], + "os_id": 535, + "app_id": 0, + "image_id": "mldev", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + }, + { + "id": "90676f7b-5a34-49a8-b770-b0abed017577", + "os": "Application", + "ram": 2048, + "disk": 64, + "main_ip": "100.68.102.45", + "vcpu_count": 1, + "region": "ord", + "plan": "vhf-1c-2gb", + "date_created": "2022-12-05T12:54:37+00:00", + "status": "active", + "allowed_bandwidth": 2000, + "netmask_v4": "255.255.254.0", + "gateway_v4": "149.28.114.1", + "power_status": "running", + "server_status": "installingbooting", + "v6_network": "2001:19f0:face:16ec::", + "v6_main_ip": "2001:19f0:face:16ec:5400:04ff:beef:cafe", + "v6_network_size": 64, + "label": "wordpress-guest", + "internal_ip": "10.2.96.3", + "kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=random-data", + "hostname": "wordpress-guest", + "tag": "", + "tags": [ + "vpc", + "vpc:d0db548b-5d50-4c94-9cdb-b083d46925f8", + "application" + ], + "os_id": 186, + "app_id": 2, + "image_id": "", + "firewall_group_id": "", + "features": [ + "auto_backups", + "ipv6" + ] + } + ], + "meta": { + "total": 8, + "links": { + "next": "", + "prev": "page1" + } + } +} diff --git a/tests/unit/plugins/inventory/test_vultr.py b/tests/unit/plugins/inventory/test_vultr.py new file mode 100644 index 0000000..aa8d8fb --- /dev/null +++ b/tests/unit/plugins/inventory/test_vultr.py @@ -0,0 +1,233 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import json +import os.path +import pytest + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.inventory.data import InventoryData +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar + +import ansible_collections.vultr.cloud.plugins.inventory.vultr as module_under_test +from ansible_collections.vultr.cloud.plugins.inventory.vultr import InventoryModule + + +default_options = { + "api_endpoint": "https://test.api.vultr.com/v2", + "api_key": "TEST_VULTR_API_KEY", + "api_results_per_page": 100, + "api_timeout": 60, + "attributes": ["id", "region", "label", "plan", "hostname", "main_ip"], + "filters": [], + "plugin": "vultr.cloud.vultr", + "variable_prefix": "vultr_", + "validate_certs": True, +} + + +def get_option(opts): + def func(option): + return opts.get(option, False) + + return func + + +def load_fixture(filename): + return open(os.path.join(os.path.dirname(__file__), "fixtures", filename)) + + +def get_paginated_json_response(url): + cursor = "page1" + if "&cursor=" in url: + cursor = "page2" + return load_fixture("vultr_inventory_{0}.json".format(cursor)) + + +@pytest.fixture() +def inventory(): + r = InventoryModule() + r.inventory = InventoryData() + r.templar = Templar(loader=DataLoader()) + return r + + +@pytest.fixture() +def instances(): + return json.load(load_fixture("vultr_inventory.json")) + + +def test_verify_file_no_filename(inventory): + assert inventory.verify_file("") is False + + +def test_verify_file_valid_filename(tmp_path, inventory): + valid_config = tmp_path / "vultr.yaml" + valid_config.touch() + assert inventory.verify_file(str(valid_config)) is True + + +def test_verify_file_invalid_filename(tmp_path, inventory): + invalid_config = tmp_path / "vultr.notyaml" + invalid_config.touch() + assert inventory.verify_file(str(invalid_config)) is False + + +@pytest.mark.parametrize("cache_option", [True, False]) +def test_parse(tmp_path, inventory, mocker, cache_option): + inventory_file = tmp_path / "vultr.yaml" + inventory_file.write_text("---\nplugin: vultr.cloud.vultr") + + plugin_cache_dir = tmp_path / "cache" + + opts = default_options.copy() + opts.update( + { + "cache": cache_option, + "cache_connection": plugin_cache_dir, + "cache_plugin": "jsonfile", + } + ) + inventory.get_option = mocker.MagicMock(side_effect=get_option(opts)) + + mocker.patch("{0}.Request".format(module_under_test.__name__)) + RequestMock = module_under_test.Request + + req = RequestMock.return_value + req.get.side_effect = get_paginated_json_response + + inventory._redirected_names = ["vultr.cloud.vultr", "vultr"] + inventory._load_name = "vultr.cloud.vultr" + inventory.parse(inventory.inventory, DataLoader(), str(inventory_file)) + + if cache_option: + RequestMock.reset_mock() + + inventory.parse(inventory.inventory, DataLoader(), str(inventory_file)) + RequestMock.assert_not_called() + + assert len(inventory.inventory.hosts.items()) > 0 + + +def test_parse_non_plugin_invalid_parameter(inventory): + try: + inventory.parse(None, DataLoader(), "") + assert False, "Expected parse() to raise AnsibleParserError" + except AnsibleParserError: + pass + + +def test_get_instances(inventory, mocker): + inventory.get_option = mocker.MagicMock(side_effect=get_option(default_options)) + + mocker.patch("{0}.Request".format(module_under_test.__name__)) + RequestMock = module_under_test.Request + + req = RequestMock.return_value + req.get.side_effect = get_paginated_json_response + + instance_list = inventory._get_instances() + assert len(instance_list) == 8 + + +def test_get_instances_invalid_api_key(inventory, mocker): + inventory.get_option = mocker.MagicMock(side_effect=get_option(default_options)) + + mocker.patch("{0}.Request".format(module_under_test.__name__)) + RequestMock = module_under_test.Request + + req = RequestMock.return_value + req.get.return_value = load_fixture("unauthorized_vultr_inventory.json") + + try: + inventory._get_instances() + assert False, "Expected _get_instances() to raise AnsibleParserError" + except AnsibleParserError: + pass + + +def test_get_instances_templated_api_key(inventory, mocker): + opts = default_options.copy() + opts.update({"api_key": '{{ lookup("random_choice", "TEST_VULTR_API_KEY") }}'}) + + inventory.get_option = mocker.MagicMock(side_effect=get_option(opts)) + + mocker.patch("{0}.Request".format(module_under_test.__name__)) + RequestMock = module_under_test.Request + + req = RequestMock.return_value + req.get.return_value = load_fixture("empty_vultr_inventory.json") + + inventory._get_instances() + + req_headers = RequestMock.call_args.kwargs["headers"] + assert req_headers.get("Authorization") == "Bearer TEST_VULTR_API_KEY" + + +def test_populate(inventory, instances, mocker): + inventory.get_option = mocker.MagicMock(side_effect=get_option(default_options)) + inventory._populate(instances) + + assert len(inventory.inventory.hosts.items()) > 0 + + +def test_populate_with_empty_response(inventory, mocker): + inventory.get_option = mocker.MagicMock(side_effect=get_option(default_options)) + + mocker.patch("{0}.Request".format(module_under_test.__name__)) + RequestMock = module_under_test.Request + + req = RequestMock.return_value + req.get.return_value = load_fixture("empty_vultr_inventory.json") + + inventory._populate(inventory._get_instances()) + + assert len(inventory.inventory.hosts.items()) == 0 + + +def test_populate_host_variables(inventory, instances, mocker): + inventory.get_option = mocker.MagicMock(side_effect=get_option(default_options)) + inventory._populate(instances) + + windows_host = inventory.inventory.get_host("windows-guest") + assert windows_host.vars["vultr_plan"] == "vhp-1c-2gb-amd" + assert windows_host.vars["vultr_region"] == "fra" + assert windows_host.vars["vultr_id"] == "2db0bb8c-9d83-4e62-86ef-b3d999960d72" + + +def test_populate_host_variables_with_filters(inventory, instances, mocker): + opts = default_options.copy() + opts.update({"filters": ['vultr_id == "37d67f9f-17f0-4c56-879b-507355dc5174"']}) + + inventory.get_option = mocker.MagicMock(side_effect=get_option(opts)) + inventory._populate(instances) + + unfiltered_host = inventory.inventory.get_host("openbsd-guest") + filtered_host = inventory.inventory.get_host("debian-guest") + + for host in inventory.inventory.hosts: + this_host = inventory.inventory.get_host(host) + assert "vultr_id" in this_host.vars + assert this_host.vars["vultr_id"] == "37d67f9f-17f0-4c56-879b-507355dc5174" + + assert len(inventory.inventory.hosts.items()) == 1 + assert unfiltered_host.vars["vultr_plan"] == "voc-g-1c-4gb-30s-amd" + assert filtered_host is None + + +def test_passes_filters_invalid_filter_not_strict(inventory): + try: + inventory._passes_filters(["invalid filter"], {}, "host", False) + assert True + except AnsibleError: + assert False, "unexpected AnsibleError from _passes_filters()" + + +def test_passes_filters_invalid_filter_strict(inventory): + try: + inventory._passes_filters(["invalid filter"], {}, "host", True) + assert False, "expected _passes_filters() to raise AnsibleError" + except AnsibleError: + pass From ea1daf1099df08ffca5f4961009ede7d17a5f04d Mon Sep 17 00:00:00 2001 From: Jeff Sites Date: Thu, 8 Dec 2022 07:59:44 -0500 Subject: [PATCH 2/3] Run unit tests on all PRs --- .github/workflows/unit.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 4b3cbd4..80d16eb 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -3,7 +3,6 @@ name: Unit tests on: pull_request: - types: [labeled] push: branches: - main @@ -11,6 +10,8 @@ on: - '.github/workflows/unit.yml' - 'plugins/**' - 'tests/unit/**' + workflow_call: + workflow_dispatch: jobs: unit-test: From a7e9ad42691dae13507bf595199613f6d8438908 Mon Sep 17 00:00:00 2001 From: Jeff Sites Date: Thu, 8 Dec 2022 08:10:34 -0500 Subject: [PATCH 3/3] Rebase and update unit test workflow --- .github/workflows/unit.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 80d16eb..f75caba 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -17,7 +17,7 @@ jobs: unit-test: name: Unit test Ansible ${{ matrix.ansible }} Py${{ matrix.python }} if: ${{ github.event.label.name == 'automation' || github.ref_name == 'main' }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 defaults: run: working-directory: ansible_collections/vultr/cloud @@ -25,12 +25,9 @@ jobs: fail-fast: false matrix: ansible: - - stable-2.11 - - stable-2.13 + - stable-2.14 python: - - 2.7 - - 3.6 - - 3.8 + - "3.10" steps: - name: Check out code uses: actions/checkout@v3