Skip to content

Commit 54ff260

Browse files
committed
Update CPU and memory details by switching to lshw method instead of cpuinfo
Add CPU information (model, vendor, frequency) and memory details (clock, size, type) to API
1 parent a04783d commit 54ff260

File tree

10 files changed

+139
-15
lines changed

10 files changed

+139
-15
lines changed

.github/workflows/code-quality.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
run: |
1717
sudo apt-get update
1818
sudo apt-get -y upgrade
19-
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
19+
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-nftables python3-jsonschema
2020
pip install --upgrade typing-extensions types-PyYAML
2121
2222
- name: Install required Python packages

docker/vm_supervisor-dev.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ FROM debian:bullseye
55
RUN apt-get update && apt-get -y upgrade && apt-get install -y \
66
sudo acl curl squashfs-tools git \
77
python3 python3-aiohttp python3-alembic python3-msgpack python3-pip python3-aiodns python3-aioredis\
8-
python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo ndppd \
8+
python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging ndppd \
99
&& rm -rf /var/lib/apt/lists/*
1010

1111
RUN useradd jailman

packaging/aleph-vm/DEBIAN/control

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ Version: 0.1.8
33
Architecture: all
44
Maintainer: Aleph.im
55
Description: Aleph.im VM execution engine
6-
Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs
6+
Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs,lshw
77
Section: aleph-im
88
Priority: Extra

packaging/requirements-debian-11.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ multidict==5.1.0
1717
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
1818
packaging==20.9
1919
psutil==5.8.0
20-
py-cpuinfo==5.0.0
2120
pycares==3.1.1
2221
pyparsing==2.4.7
2322
pyrsistent==0.15.5

packaging/requirements-ubuntu-20.04.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ multidict==4.7.3
1818
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
1919
packaging==20.3
2020
psutil==5.5.1
21-
py-cpuinfo==5.0.0
2221
pycares==3.1.1
2322
PyGObject==3.36.0
2423
pyparsing==2.4.6

packaging/requirements-ubuntu-22.04.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ multidict==5.1.0
2121
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
2222
packaging==21.3
2323
psutil==5.9.0
24-
py-cpuinfo==5.0.0
2524
pycares==4.1.2
2625
PyGObject==3.42.1
2726
pyparsing==2.4.7

pyproject.toml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ dependencies = [
3535
"sentry-sdk==1.31.0",
3636
"aioredis==1.3.1",
3737
"psutil==5.9.5",
38-
"py-cpuinfo==9.0.0",
3938
"schedule==1.2.1",
4039
"nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
4140
"msgpack==1.0.7",
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import json
2+
import re
3+
import subprocess
4+
from functools import lru_cache
5+
6+
7+
@lru_cache
8+
def get_hardware_info():
9+
lshw = subprocess.Popen(["lshw", "-sanitize", "-json"], stdout=subprocess.PIPE, shell=False)
10+
output, _ = lshw.communicate()
11+
data = json.loads(output)
12+
13+
hw_info = {}
14+
15+
memory_bank0 = None
16+
17+
for hw in data["children"][0]["children"]:
18+
if hw["id"] == "cpu":
19+
hw_info["cpu"] = hw
20+
elif hw["class"] == "memory" and hw["id"] == "memory":
21+
hw_info["memory"] = hw
22+
23+
hw_info["memory"]["bank:0"] = memory_bank0
24+
return hw_info
25+
26+
27+
@lru_cache
28+
def get_cpu_info():
29+
hw = get_hardware_info()
30+
31+
cpu_info = hw["cpu"]
32+
architecture = cpu_info["width"]
33+
34+
if "x86_64" in cpu_info["capabilities"] or "x86-64" in cpu_info["capabilities"]:
35+
architecture = "x86_64"
36+
elif "arm64" in cpu_info["capabilities"] or "arm-64" in cpu_info["capabilities"]:
37+
architecture = "arm64"
38+
39+
vendor = cpu_info["vendor"]
40+
# lshw vendor implementation => https://github.com/lyonel/lshw/blob/15e4ca64647ad119b69be63274e5de2696d3934f/src/core/cpuinfo.cc#L308
41+
42+
if "Intel Corp" in vendor:
43+
vendor = "GenuineIntel"
44+
elif "Advanced Micro Devices [AMD]" in vendor:
45+
vendor = "AuthenticAMD"
46+
47+
return {
48+
"architecture": architecture,
49+
"vendor": vendor,
50+
"model": cpu_info["product"],
51+
"frequency": cpu_info["capacity"],
52+
}
53+
54+
55+
@lru_cache
56+
def get_memory_info():
57+
hw = get_hardware_info()
58+
mem_info = hw["memory"]
59+
60+
memory_type = ""
61+
memory_clock = ""
62+
63+
for bank in mem_info["children"]:
64+
memory_clock = bank["clock"]
65+
try:
66+
memory_type = re.search("(DDR[2-6])", bank["description"]).group(0)
67+
break
68+
except:
69+
pass
70+
71+
return {
72+
"size": mem_info["size"],
73+
"units": mem_info["units"],
74+
"type": memory_type,
75+
"clock": memory_clock,
76+
"clock_units": "Hz",
77+
}

src/aleph/vm/orchestrator/resources.py

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,14 @@
33
from functools import lru_cache
44
from typing import Optional
55

6-
import cpuinfo
76
import psutil
87
from aiohttp import web
8+
from aleph.vm.conf import settings
9+
from aleph.vm.orchestrator.machine import get_cpu_info, get_memory_info
910
from aleph_message.models import ItemHash
1011
from aleph_message.models.execution.environment import CpuProperties
1112
from pydantic import BaseModel, Field
1213

13-
from aleph.vm.conf import settings
14-
1514

1615
class Period(BaseModel):
1716
datetime: datetime
@@ -76,18 +75,63 @@ class MachineUsage(BaseModel):
7675
active: bool = True
7776

7877

78+
class ExtendedCpuProperties(CpuProperties):
79+
"""CPU properties."""
80+
81+
model: Optional[str] = Field(default=None, description="CPU model")
82+
frequency: Optional[str] = Field(default=None, description="CPU frequency")
83+
84+
85+
class MemoryProperties(BaseModel):
86+
"""MEMORY properties."""
87+
88+
size: Optional[str] = Field(default=None, description="Memory size")
89+
units: Optional[str] = Field(default=None, description="Memory size units")
90+
type: Optional[str] = Field(default=None, description="Memory type")
91+
clock: Optional[str] = Field(default=None, description="Memory clock")
92+
clock_units: Optional[str] = Field(default=None, description="Memory clock units")
93+
94+
95+
class MachineCapability(BaseModel):
96+
cpu: ExtendedCpuProperties
97+
memory: MemoryProperties
98+
99+
79100
@lru_cache
80101
def get_machine_properties() -> MachineProperties:
81102
"""Fetch machine properties such as architecture, CPU vendor, ...
82103
These should not change while the supervisor is running.
83104
84105
In the future, some properties may have to be fetched from within a VM.
85106
"""
86-
cpu_info = cpuinfo.get_cpu_info() # Slow
107+
108+
cpu_info = get_cpu_info()
87109
return MachineProperties(
88110
cpu=CpuProperties(
89-
architecture=cpu_info["raw_arch_string"],
90-
vendor=cpu_info["vendor_id"],
111+
architecture=cpu_info["architecture"],
112+
vendor=cpu_info["vendor"],
113+
),
114+
)
115+
116+
117+
@lru_cache
118+
def get_machine_capability() -> MachineCapability:
119+
cpu_info = get_cpu_info()
120+
mem_info = get_memory_info()
121+
122+
return MachineCapability(
123+
cpu=ExtendedCpuProperties(
124+
architecture=cpu_info["architecture"],
125+
vendor=cpu_info["vendor"],
126+
model=cpu_info["model"],
127+
frequency=cpu_info["frequency"],
128+
),
129+
memory=MemoryProperties(
130+
size=mem_info["size"],
131+
units=mem_info["units"],
132+
type=mem_info["type"],
133+
clock=mem_info["clock"],
134+
clock_units=mem_info["clock_units"],
91135
),
92136
)
93137

@@ -119,6 +163,13 @@ async def about_system_usage(_: web.Request):
119163
return web.json_response(text=usage.json(exclude_none=True), headers={"Access-Control-Allow-Origin:": "*"})
120164

121165

166+
async def about_capability(_: web.Request):
167+
"""Public endpoint to expose information about the CRN capability."""
168+
169+
capability: MachineCapability = get_machine_capability()
170+
return web.json_response(text=capability.json(exclude_none=False), headers={"Access-Control-Allow-Origin:": "*"})
171+
172+
122173
class Allocation(BaseModel):
123174
"""An allocation is the set of resources that are currently allocated on this orchestrator.
124175
It contains the item_hashes of all persistent VMs, instances, on-demand VMs and jobs.

src/aleph/vm/orchestrator/supervisor.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,12 @@
1414
from typing import Callable
1515

1616
from aiohttp import web
17-
1817
from aleph.vm.conf import settings
1918
from aleph.vm.pool import VmPool
2019
from aleph.vm.version import __version__
2120

2221
from .metrics import create_tables, setup_engine
23-
from .resources import about_system_usage
22+
from .resources import about_capability, about_system_usage
2423
from .tasks import (
2524
start_payment_monitoring_task,
2625
start_watch_for_messages_task,
@@ -93,6 +92,7 @@ async def allow_cors_on_endpoint(request: web.Request):
9392
web.get("/about/executions/records", about_execution_records),
9493
web.get("/about/usage/system", about_system_usage),
9594
web.get("/about/config", about_config),
95+
web.get("/about/capability", about_capability),
9696
# /control APIs are used to control the VMs and access their logs
9797
web.post("/control/allocations", update_allocations),
9898
web.post("/control/allocation/notify", notify_allocation),

0 commit comments

Comments
 (0)