Allow validation of Openshift allocations when changing quotas

As part of allowing the validation of allocation change
requests (acr) [1], various functions have been added to the
plugin that are mostly called from Coldfront to validate an acr.
Among these changes are:

- Several new functions to `utils.py` to perform basic tasks
- Functions in the openshift allocator to obtain the usage of a Openshift project
- Refactoring of how the Openshift quota value is parsed
- New unit tests

[1] nerc-project/coldfront-nerc#138

Author: QuanMPhm

src/coldfront_plugin_cloud/base.py

@@ -77,3 +77,7 @@ def assign_role_on_user(self, username, project_id):
     @abc.abstractmethod
     def remove_role_from_user(self, username, project_id):
         pass
+
+    @abc.abstractmethod
+    def get_usage(self, project_id):
+        pass

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -1,13 +1,12 @@
 import logging
-import re
 
 from coldfront_plugin_cloud import attributes
 from coldfront_plugin_cloud import openstack
 from coldfront_plugin_cloud import openshift
 from coldfront_plugin_cloud import utils
 from coldfront_plugin_cloud import tasks
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 from coldfront.core.resource.models import Resource
 from coldfront.core.allocation.models import (
     Allocation,
@@ -242,51 +241,9 @@ def handle(self, *args, **options):
 
                 expected_value = allocation.get_attribute(attr)
                 current_value = quota.get(key, None)
-
-                PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?"
-
-                suffix = {
-                    "Ki": 2**10,
-                    "Mi": 2**20,
-                    "Gi": 2**30,
-                    "Ti": 2**40,
-                    "Pi": 2**50,
-                    "Ei": 2**60,
-                    "m": 10**-3,
-                    "K": 10**3,
-                    "M": 10**6,
-                    "G": 10**9,
-                    "T": 10**12,
-                    "P": 10**15,
-                    "E": 10**18,
-                }
-
-                if current_value and current_value != "0":
-                    result = re.search(PATTERN, current_value)
-
-                    if result is None:
-                        raise CommandError(
-                            f"Unable to parse current_value = '{current_value}' for {attr}"
-                        )
-
-                    value = int(result.groups()[0])
-                    unit = result.groups()[1]
-
-                    # Convert to number i.e. without any unit suffix
-
-                    if unit is not None:
-                        current_value = value * suffix[unit]
-                    else:
-                        current_value = value
-
-                    # Convert some attributes to units that coldfront uses
-
-                    if "RAM" in attr:
-                        current_value = round(current_value / suffix["Mi"])
-                    elif "Storage" in attr:
-                        current_value = round(current_value / suffix["Gi"])
-                elif current_value and current_value == "0":
-                    current_value = 0
+                current_value = openshift.parse_openshift_quota_value(
+                    attr, current_value
+                )
 
                 if expected_value is None and current_value is not None:
                     msg = (

src/coldfront_plugin_cloud/openshift.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import re
 import requests
 from requests.auth import HTTPBasicAuth
 import time
@@ -44,6 +45,54 @@ def clean_openshift_metadata(obj):
     return obj
 
 
+def parse_openshift_quota_value(attr_name, quota_value):
+    PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?"
+
+    suffix = {
+        "Ki": 2**10,
+        "Mi": 2**20,
+        "Gi": 2**30,
+        "Ti": 2**40,
+        "Pi": 2**50,
+        "Ei": 2**60,
+        "m": 10**-3,
+        "K": 10**3,
+        "M": 10**6,
+        "G": 10**9,
+        "T": 10**12,
+        "P": 10**15,
+        "E": 10**18,
+    }
+
+    if quota_value and quota_value != "0":
+        result = re.search(PATTERN, quota_value)
+
+        if result is None:
+            raise ValueError(
+                f"Unable to parse quota_value = '{quota_value}' for {attr_name}"
+            )
+
+        value = int(result.groups()[0])
+        unit = result.groups()[1]
+
+        # Convert to number i.e. without any unit suffix
+
+        if unit is not None:
+            quota_value = value * suffix[unit]
+        else:
+            quota_value = value
+
+        # Convert some attributes to units that coldfront uses
+
+        if "RAM" in attr_name:
+            return round(quota_value / suffix["Mi"])
+        elif "Storage" in attr_name:
+            return round(quota_value / suffix["Gi"])
+        return quota_value
+    elif quota_value and quota_value == "0":
+        return 0
+
+
 class ApiException(Exception):
     def __init__(self, message):
         self.message = message
@@ -69,6 +118,11 @@ class OpenShiftResourceAllocator(base.ResourceAllocator):
         attributes.QUOTA_PVC: lambda x: {"persistentvolumeclaims": f"{x}"},
     }
 
+    CF_QUOTA_KEY_MAPPING = {
+        list(quota_lambda_func(0).keys())[0]: cf_attr
+        for cf_attr, quota_lambda_func in QUOTA_KEY_MAPPING.items()
+    }
+
     resource_type = "openshift"
 
     project_name_max_length = 63
@@ -200,6 +254,21 @@ def get_quota(self, project_id):
 
         return combined_quota
 
+    def get_usage(self, project_id):
+        cloud_quotas = self._openshift_get_resourcequotas(project_id)
+        combined_quota_used = {}
+        for cloud_quota in cloud_quotas:
+            combined_quota_used.update(
+                {
+                    self.CF_QUOTA_KEY_MAPPING[quota_key]: parse_openshift_quota_value(
+                        self.CF_QUOTA_KEY_MAPPING[quota_key], value
+                    )
+                    for quota_key, value in cloud_quota["status"]["used"].items()
+                }
+            )
+
+        return combined_quota_used
+
     def create_project_defaults(self, project_id):
         pass
 

src/coldfront_plugin_cloud/openstack.py

@@ -470,3 +470,6 @@ def get_users(self, project_id):
             role_assignment.user["name"] for role_assignment in role_assignments
         )
         return user_names
+
+    def get_usage(self, project_id):
+        raise NotImplementedError

src/coldfront_plugin_cloud/tasks.py

@@ -191,3 +191,27 @@ def remove_user_from_allocation(allocation_user_pk):
             allocator.remove_role_from_user(username, project_id)
         else:
             logger.warning("No project has been created. Nothing to disable.")
+
+
+def get_allocation_cloud_usage(allocation_pk):
+    """
+    Obtains the current quota usage for the allocation.
+
+    For example, the output for an Openshift quota would be:
+
+    {
+        "limits.cpu": "1",
+        "limits.memory": "2Gi",
+        "limits.ephemeral-storage": "10Gi",
+    }
+    """
+    allocation = Allocation.objects.get(pk=allocation_pk)
+    if allocator := find_allocator(allocation):
+        if project_id := allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID):
+            try:
+                return allocator.get_usage(project_id)
+            except NotImplemented:
+                return
+        else:
+            logger.warning("No project has been created. No quota to check.")
+            return

src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py

@@ -109,3 +109,25 @@ def test_get_moc_quota(self, fake_get_quota):
         ]
         res = self.allocator.get_quota("fake-project")
         self.assertEqual(res, expected_quota)
+
+    @mock.patch(
+        "coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._openshift_get_resourcequotas"
+    )
+    def test_get_moc_usage(self, fake_get_quota):
+        fake_usage = {
+            "limits.cpu": "2000m",
+            "limits.memory": "4096Mi",
+        }
+        fake_get_quota.return_value = [
+            {
+                "status": {"used": fake_usage},
+            }
+        ]
+        res = self.allocator.get_usage("fake-project")
+        self.assertEqual(
+            res,
+            {
+                "OpenShift Limit on CPU Quota": 2.0,
+                "OpenShift Limit on RAM Quota (MiB)": 4096,
+            },
+        )

src/coldfront_plugin_cloud/tests/unit/test_utils.py

@@ -29,3 +29,54 @@ def test_env_safe_name(self):
         self.assertEqual(utils.env_safe_name(42), "42")
         self.assertEqual(utils.env_safe_name(None), "NONE")
         self.assertEqual(utils.env_safe_name("hello"), "HELLO")
+
+
+class TestCheckIfQuotaAttr(base.TestCase):
+    def test_valid_quota_attr(self):
+        self.assertTrue(utils.check_if_quota_attr("OpenShift Limit on CPU Quota"))
+
+    def test_invalid_quota_attr(self):
+        self.assertFalse(utils.check_if_quota_attr("Test"))
+        self.assertFalse(utils.check_if_quota_attr("Allocated Project ID"))
+
+
+class TestGetNewCloudQuota(base.TestCase):
+    def test_get_requested_quota(self):
+        data = [
+            {"name": "OpenShift Limit on CPU Quota", "new_value": "2"},
+            {"name": "OpenShift Limit on RAM Quota (MiB)", "new_value": ""},
+        ]
+
+        result = utils.get_new_cloud_quota(data)
+        self.assertEqual(result, {"OpenShift Limit on CPU Quota": "2"})
+
+
+class TestCheckChangeRequests(base.TestBase):
+    def test_check_usage(self):
+        # No error case, usage is lower
+        test_quota_usage = {
+            "limits.cpu": "1",
+            "limits.memory": "2Gi",
+            "limits.ephemeral-storage": "10Gi",  # Other quotas should be ignored
+            "requests.storage": "40Gi",
+            "requests.nvidia.com/gpu": "0",
+            "persistentvolumeclaims": "4",
+        }
+        test_requested_quota = {"OpenShift Limit on CPU Quota": "2"}
+
+        self.assertEqual(
+            [], utils.check_cloud_usage_is_lower(test_requested_quota, test_quota_usage)
+        )
+
+        # Requested cpu (2) lower than current used, should return errors
+        test_quota_usage["limits.cpu"] = "16"
+        self.assertEqual(
+            [
+                (
+                    "Current quota usage for OpenShift Limit on CPU Quota "
+                    "(16) is higher than "
+                    "the requested amount (2)."
+                )
+            ],
+            utils.check_cloud_usage_is_lower(test_requested_quota, test_quota_usage),
+        )

src/coldfront_plugin_cloud/utils.py

@@ -244,3 +244,53 @@ def get_included_duration(
         total_interval_duration -= (e_interval_end - e_interval_start).total_seconds()
 
     return math.ceil(total_interval_duration)
+
+
+def check_if_quota_attr(attr_name: str):
+    for quota_attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
+        if attr_name == quota_attr.name:
+            return True
+    return False
+
+
+def get_new_cloud_quota(change_request_data: list[dict[str, str]]):
+    """
+    Converts change request data to a dictionary of requested quota changes.
+    Ignores attributes with empty `new_value` str, meaning no change requested for them
+    Input typically looks like:
+    [
+        {
+            "name": "OpenShift Limit on CPU Quota",
+            "new_value": "2",
+            ...
+        },
+        {
+            "name": "OpenShift Limit on RAM Quota (MiB)",
+            "new_value": "",
+            ...
+        }
+    ]
+    """
+    requested_quota = {}
+    for form in change_request_data:
+        if check_if_quota_attr(form["name"]) and form["new_value"]:
+            requested_quota[form["name"]] = form["new_value"]
+    return requested_quota
+
+
+def check_cloud_usage_is_lower(
+    requested_quota: dict[str, str], cloud_quota_usage: dict[str, str]
+):
+    usage_errors = []
+    for quota_name, requested_quota_value in requested_quota.items():
+        current_usage_value = cloud_quota_usage[quota_name]
+        if int(requested_quota_value) < current_usage_value:
+            usage_errors.append(
+                (
+                    f"Current quota usage for {quota_name} "
+                    f"({current_usage_value}) is higher than "
+                    f"the requested amount ({requested_quota_value})."
+                )
+            )
+
+    return usage_errors