Fix tests and parsing

Author: olethanh

src/aleph/vm/orchestrator/machine.py

@@ -1,17 +1,18 @@
 import asyncio
 import json
 import re
+import shutil
 
 import psutil
 
+from aleph.vm.utils import run_in_subprocess
 
-async def get_hardware_info():
-    lshw = await asyncio.create_subprocess_shell(
-        "lshw -sanitize -json", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-    )
 
-    output, _ = await lshw.communicate()
-    data = json.loads(output)
+async def get_hardware_info():
+    lshw_path = shutil.which("lshw")
+    assert lshw_path, "lshw not found in PATH. apt install lshw."
+    lshw_output = await run_in_subprocess([lshw_path, "-sanitize", "-json"])
+    data = json.loads(lshw_output)
 
     hw_info = {"cpu": None, "memory": None}
 
@@ -26,12 +27,13 @@ async def get_hardware_info():
 
 def get_cpu_info(hw):
     cpu_info = hw["cpu"]
-    architecture = cpu_info["width"]
 
     if "x86_64" in cpu_info["capabilities"] or "x86-64" in cpu_info["capabilities"]:
         architecture = "x86_64"
     elif "arm64" in cpu_info["capabilities"] or "arm-64" in cpu_info["capabilities"]:
         architecture = "arm64"
+    else:
+        architecture = None
 
     vendor = cpu_info["vendor"]
     # lshw vendor implementation => https://github.com/lyonel/lshw/blob/15e4ca64647ad119b69be63274e5de2696d3934f/src/core/cpuinfo.cc#L308

src/aleph/vm/orchestrator/resources.py

@@ -99,17 +99,17 @@ class ExtendedCpuProperties(CpuProperties):
     """CPU properties."""
 
     model: Optional[str] = Field(default=None, description="CPU model")
-    frequency: Optional[str] = Field(default=None, description="CPU frequency")
-    count: Optional[str] = Field(default=None, description="CPU count")
+    frequency: Optional[int] = Field(default=None, description="CPU frequency")
+    count: Optional[int] = Field(default=None, description="CPU count")
 
 
 class MemoryProperties(BaseModel):
     """MEMORY properties."""
 
-    size: Optional[str] = Field(default=None, description="Memory size")
+    size: Optional[int] = Field(default=None, description="Memory size")
     units: Optional[str] = Field(default=None, description="Memory size units")
     type: Optional[str] = Field(default=None, description="Memory type")
-    clock: Optional[str] = Field(default=None, description="Memory clock")
+    clock: Optional[int] = Field(default=None, description="Memory clock")
     clock_units: Optional[str] = Field(default=None, description="Memory clock units")
 
 
@@ -170,8 +170,8 @@ async def get_machine_capability() -> MachineCapability:
             architecture=cpu_info["architecture"],
             vendor=cpu_info["vendor"],
             model=cpu_info["model"],
-            frequency=cpu_info["frequency"],
-            count=cpu_info["count"],
+            frequency=(cpu_info["frequency"]),
+            count=(cpu_info["count"]),
             features=list(
                 filter(
                     None,

src/aleph/vm/pool.py

@@ -405,7 +405,7 @@ async def reserve_resources(self, message: ExecutableContent, user):
         return expiration_date
 
     def find_resources_available_for_user(self, message: ExecutableContent, user) -> set[GpuDevice]:
-        """Find required resource to run ExecutableContent from reserved resources by user or free resources.
+        """Find the required resource to run ExecutableContent from reserved resources by user or free resources.
 
         Only implement GPU for now"""
         # Calling function should use the creation_lock to avoid resource being stollem

tests/supervisor/test_views.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import tempfile
 from copy import deepcopy
 from pathlib import Path
@@ -11,7 +12,6 @@
 from pytest_mock import MockerFixture
 
 from aleph.vm.conf import settings
-from aleph.vm.orchestrator.machine import get_hardware_info
 from aleph.vm.orchestrator.supervisor import setup_webapp
 from aleph.vm.pool import VmPool
 from aleph.vm.sevclient import SevClient
@@ -45,6 +45,7 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
 @pytest.mark.asyncio
 async def test_system_usage(aiohttp_client, mocker, mock_app_with_pool):
     """Test that the usage system endpoints responds. No auth needed"""
+    mocker.patch("aleph.vm.orchestrator.resources.get_hardware_info", return_value=MOCK_SYSTEM_INFO)
 
     client = await aiohttp_client(await mock_app_with_pool)
     response: web.Response = await client.get("/about/usage/system")
@@ -55,7 +56,7 @@ async def test_system_usage(aiohttp_client, mocker, mock_app_with_pool):
     assert resp["cpu"]["count"] > 0
 
 
-FAKE_SYSTEM_INFO = {
+MOCK_SYSTEM_INFO = {
     "cpu": {
         "id": "cpu",
         "class": "processor",
@@ -74,6 +75,7 @@ async def test_system_usage(aiohttp_client, mocker, mock_app_with_pool):
         "width": 64,
         "configuration": {"cores": "8", "enabledcores": "8", "microcode": "167776681", "threads": "1"},
         "capabilities": {
+            "x86-64": "64bits extensions (x86-64)",
             "fpu": "mathematical co-processor",
             "fpu_exception": "FPU exceptions reporting",
             "wp": True,
@@ -209,7 +211,7 @@ async def test_system_usage(aiohttp_client, mocker, mock_app_with_pool):
 async def test_system_usage_mock(aiohttp_client, mocker, mock_app_with_pool):
     """Test that the usage system endpoints response value. No auth needed"""
 
-    mocker.patch("aleph.vm.orchestrator.machine.get_hardware_info", FAKE_SYSTEM_INFO)
+    mocker.patch("aleph.vm.orchestrator.resources.get_hardware_info", return_value=MOCK_SYSTEM_INFO)
     mocker.patch(
         "psutil.getloadavg",
         lambda: [1, 2, 3],
@@ -233,7 +235,10 @@ async def test_system_usage_mock(aiohttp_client, mocker, mock_app_with_pool):
 @pytest.mark.asyncio
 async def test_system_capability_mock(aiohttp_client, mocker):
     """Test that the capability system endpoints response value. No auth needed"""
-    mocker.patch("aleph.vm.orchestrator.machine.get_hardware_info", FAKE_SYSTEM_INFO)
+    mocker.patch("aleph.vm.orchestrator.resources.get_hardware_info", return_value=MOCK_SYSTEM_INFO)
+    mocker.patch("aleph.vm.orchestrator.resources.check_amd_sev_supported", return_value=True)
+    mocker.patch("aleph.vm.orchestrator.resources.check_amd_sev_es_supported", return_value=True)
+    mocker.patch("aleph.vm.orchestrator.resources.check_amd_sev_snp_supported", return_value=False)
     mocker.patch(
         "psutil.getloadavg",
         lambda: [1, 2, 3],
@@ -252,14 +257,34 @@ async def test_system_capability_mock(aiohttp_client, mocker):
         "cpu": {
             "architecture": "x86_64",
             "vendor": "AuthenticAMD",
+            "features": ["sev", "sev_es"],
             "model": "AMD EPYC 7763 64-Core Processor",
-            "frequency": "2000000000",
-            "count": "200",
+            "frequency": 2000000000,
+            "count": 200,
         },
-        "memory": {"size": "17179869184", "units": "bytes", "type": "", "clock": None, "clock_units": ""},
+        "memory": {"size": 17179869184, "units": "bytes", "type": "", "clock": None, "clock_units": None},
     }
 
 
+@pytest.mark.asyncio
+async def test_system_capability_real(aiohttp_client, mocker):
+    """Test that the capability system endpoints response value
+    with real system value, no mock so we don't know the definive value but want ot see that it works"""
+    if os.environ.get("GITHUB_JOB"):
+        pytest.xfail("Test fail inside GITHUB CI because of invalid lshw return inside worker")
+
+    app = setup_webapp()
+    client = await aiohttp_client(app)
+    response: web.Response = await client.get("/about/capability")
+    assert response.status == 200
+    # check if it is valid json
+    resp = await response.json()
+    assert resp.get("cpu"), resp
+    assert resp["cpu"].get("architecture")
+    assert resp.get("memory")
+    assert resp["memory"].get("size")
+
+
 @pytest.mark.asyncio
 async def test_allocation_invalid_auth_token(aiohttp_client):
     """Test that the allocation endpoint fails when an invalid auth token is provided."""
@@ -456,6 +481,7 @@ def mock_is_kernel_enabled_gpu(pci_host: str) -> bool:
 async def test_system_usage_gpu_ressources(aiohttp_client, mocker, mock_app_with_pool):
     """Test gpu are properly listed"""
     client = await aiohttp_client(await mock_app_with_pool)
+    mocker.patch("aleph.vm.orchestrator.resources.get_hardware_info", return_value=MOCK_SYSTEM_INFO)
 
     response: web.Response = await client.get("/about/usage/system")
     assert response.status == 200