Skip to content

Commit 21eb9d9

Browse files
basnijholtbasnijholt
authored
Merge ed5b269 into 75f8a6b 
2 parents 75f8a6b + ed5b269 commit 21eb9d9

File tree

5 files changed

+1017
-8
lines changed

5 files changed

+1017
-8
lines changed

‎.github/workflows/update-readme.yml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Update README.md
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
jobs:
10+
update_readme:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: Check out repository
14+
uses: actions/checkout@v3
15+
with:
16+
persist-credentials: false
17+
fetch-depth: 0
18+
19+
- name: Set up Python
20+
uses: actions/setup-python@v3
21+
with:
22+
python-version: '3.x'
23+
24+
- name: Install Python dependencies
25+
run: |
26+
python -m pip install --upgrade pip
27+
pip install markdown-code-runner
28+
pip install -e .
29+
30+
- name: Install d2
31+
run: |
32+
curl -fsSL https://d2lang.com/install.sh | sh -s --
33+
34+
- name: Run markdown-code-runner
35+
run: markdown-code-runner README.md
36+
37+
- name: Commit updated README.md
38+
id: commit
39+
run: |
40+
git add -u .
41+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
42+
git config --local user.name "github-actions[bot]"
43+
if git diff --quiet && git diff --staged --quiet; then
44+
echo "No changes in README.md, skipping commit."
45+
echo "commit_status=skipped" >> $GITHUB_ENV
46+
else
47+
git commit -m "Update README.md"
48+
echo "commit_status=committed" >> $GITHUB_ENV
49+
fi
50+
51+
- name: Push changes
52+
if: env.commit_status == 'committed'
53+
uses: ad-m/github-push-action@master
54+
with:
55+
github_token: ${{ secrets.GITHUB_TOKEN }}
56+
branch: ${{ github.head_ref }}

‎README.md

+9
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ This is an asynchronous job scheduler for [`Adaptive`](https://github.com/python
1818
- [:dart: Design Goals](#dart-design-goals)
1919
- [:test_tube: How does it work?](#test_tube-how-does-it-work)
2020
- [:mag: But how does it *really* work?](#mag-but-how-does-it-really-work)
21+
- [:bar_chart: Flow diagram](#bar_chart-flow-diagram)
2122
- [:notebook: Jupyter Notebook Example](#notebook-jupyter-notebook-example)
2223
- [:computer: Installation](#computer-installation)
2324
- [:hammer_and_wrench: Development](#hammer_and_wrench-development)
@@ -147,6 +148,14 @@ And use `scheduler.cancel(job_names)` to cancel the jobs.
147148

148149
You don't actually ever have to leave the Jupyter notebook; take a look at the [`example notebook`](https://github.com/basnijholt/adaptive-scheduler/blob/main/example.ipynb).
149150

151+
## :bar_chart: Flow diagram
152+
153+
<!-- CODE:BASH:START -->
154+
<!-- d2 docs/diagram.d2 docs/source/_static/diagram.svg -->
155+
<!-- CODE:END -->
156+
157+
![](./docs/source/_static/diagram.svg)
158+
150159
## :notebook: Jupyter Notebook Example
151160

152161
See [`example.ipynb`](https://github.com/basnijholt/adaptive-scheduler/blob/main/example.ipynb).

‎adaptive_scheduler/_scheduler/slurm.py

+13-8
Original file line numberDiff line numberDiff line change
@@ -225,11 +225,16 @@ def start_job(self, name: str) -> None:
225225
def queue(self, *, me_only: bool = True) -> dict[str, dict[str, str]]:
226226
"""Get the queue of jobs."""
227227
python_format = {
228-
"jobid": 100,
229-
"name": 100,
230-
"state": 100,
231-
"numnodes": 100,
232-
"reasonlist": 4000,
228+
"JobID": 100,
229+
"Name": 100,
230+
"State": 100,
231+
"NumNodes": 100,
232+
"NumTasks": 100,
233+
"ReasonList": 4000,
234+
"SubmitTime": 100,
235+
"StartTime": 100,
236+
"UserName": 100,
237+
"Partition": 100,
233238
} # (key -> length) mapping
234239

235240
slurm_format = ",".join(f"{k}:{v}" for k, v in python_format.items())
@@ -263,10 +268,10 @@ def line_to_dict(line: str) -> dict[str, str]:
263268

264269
squeue = [line_to_dict(line) for line in output.split("\n")]
265270
states = ("PENDING", "RUNNING", "CONFIGURING")
266-
squeue = [info for info in squeue if info["state"] in states]
267-
running = {info.pop("jobid"): info for info in squeue}
271+
squeue = [info for info in squeue if info["State"] in states]
272+
running = {info.pop("JobID"): info for info in squeue}
268273
for info in running.values():
269-
info["job_name"] = info.pop("name")
274+
info["job_name"] = info.pop("Name")
270275
return running
271276

272277
@cached_property

‎docs/diagram.d2

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
Adaptive Scheduler: {
2+
near: top-center
3+
shape: text
4+
style.font-size: 45
5+
}
6+
7+
Cluster: {
8+
icon: https://icons.terrastruct.com/tech/022-server.svg
9+
}
10+
Cluster.SLURM: {
11+
shape: queue
12+
icon: https://icons.terrastruct.com/azure%2FManagement%20and%20Governance%20Service%20Color%2FScheduler%20Job%20Collections.svg
13+
}
14+
15+
Jupyter notebook: {
16+
icon: https://icons.terrastruct.com/dev%2Fpython.svg
17+
scheduler: BaseScheduler {
18+
shape: class
19+
queue(): "list[str, dict[str, Any]]"
20+
"write_job_script(name: str, launcher_options: dict[str, Any])": None
21+
}
22+
23+
dbmanager: DataBaseManager {
24+
shape: class
25+
url: str
26+
db_fname: str
27+
learners: "list[BaseLearner]"
28+
fnames: "list[str | Path]"
29+
scheduler: BaseScheduler
30+
as_dicts(): "list[dict]"
31+
is_done(): bool
32+
}
33+
jobmanager: JobManager {
34+
shape: class
35+
job_names: "list[str]"
36+
interval: int
37+
database_manager: DataBaseManager
38+
scheduler: BaseScheduler
39+
}
40+
runmanager: RunManager {
41+
shape: class
42+
learners: "list[BaseLearner]"
43+
fnames: "list[str | Path]"
44+
job_manager: JobManager
45+
scheduler: BaseScheduler
46+
db_manager: DataBaseManager
47+
start(): None
48+
}
49+
50+
scheduler -> dbmanager: Talks to the SLURM and tracks the jobs that are running
51+
scheduler -> jobmanager: Talks to the SLURM and DataBaseManager and submits new jobs
52+
53+
runmanager -> scheduler
54+
runmanager -> dbmanager
55+
runmanager -> jobmanager
56+
}
57+
58+
Jupyter notebook.scheduler -> Cluster.SLURM: Talks to the SLURM cluster
59+
60+
Worker node: {
61+
icon: https://icons.terrastruct.com/azure%2FDatabases%20Service%20Color%2FVirtual%20Clusters.svg
62+
launcher: "laucher.py" {
63+
icon: https://icons.terrastruct.com/dev%2Fpython.svg
64+
"zmq.Socket"
65+
"adaptive.BalancingLearner"
66+
learners
67+
fnames
68+
"adaptive.Runner" {
69+
shape: class
70+
learner: BaseLearner
71+
fname: str | Path
72+
start_periodic_saving(): asyncio.Task
73+
}
74+
"zmq.Socket" -> learners
75+
"zmq.Socket" -> fnames
76+
learners -> "adaptive.BalancingLearner"
77+
fnames -> "adaptive.BalancingLearner"
78+
"adaptive.BalancingLearner" -> "adaptive.Runner"
79+
}
80+
}
81+
82+
Worker node.launcher."zmq.Socket" -> Jupyter notebook.dbmanager: "gets learners and fnames"
83+
Worker node.launcher."zmq.Socket" <- Jupyter notebook.dbmanager: "tell_done()"

0 commit comments

Comments
 (0)