Skip to content

Commit da768ef

Browse files
benoitcbenoitc
committed
Add distributed Python execution docs and Docker demo
- Add docs/distributed.md with patterns for running Python across Erlang nodes, including Elixir examples - Add examples/distributed_python.erl demonstrating parallel map, distributed inference, and pipeline processing - Add Docker setup for testing with 3-node cluster - Update getting-started.md with link to distributed docs
1 parent bd2e37f commit da768ef

File tree

9 files changed

+1359
-1
lines changed

9 files changed

+1359
-1
lines changed

‎.dockerignore‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Build artifacts - IMPORTANT: exclude to avoid cmake cache conflicts
2+
_build/
3+
*.beam
4+
*.o
5+
*.so
6+
erl_crash.dump
7+
8+
# IDE
9+
.idea/
10+
.vscode/
11+
*.swp
12+
*.swo
13+
14+
# OS files
15+
.DS_Store
16+
Thumbs.db
17+
18+
# Test artifacts
19+
logs/
20+
ct_logs/
21+
cover/
22+
benchmark_results/
23+
24+
# Git
25+
.git/
26+
.gitignore
27+
28+
# Keep docker scripts but not Dockerfiles
29+
docker/Dockerfile.*
30+
docker/docker-compose*.yml

‎docker/Dockerfile.distributed‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Dockerfile for distributed Python testing
2+
# Lightweight image for running multiple Erlang nodes with Python
3+
FROM erlang:27-slim
4+
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# Install Python 3.12 and build dependencies
8+
RUN apt-get update && apt-get install -y \
9+
build-essential \
10+
cmake \
11+
python3 \
12+
python3-dev \
13+
python3-pip \
14+
python3-venv \
15+
&& rm -rf /var/lib/apt/lists/*
16+
17+
# Install numpy for the examples
18+
RUN pip3 install --break-system-packages numpy
19+
20+
# Set environment
21+
ENV PYTHON_CONFIG=/usr/bin/python3-config
22+
23+
# Create working directory
24+
WORKDIR /app
25+
26+
# Copy source
27+
COPY . /app
28+
29+
# Ensure demo script is executable
30+
RUN chmod +x /app/start-distributed-demo.sh || true
31+
32+
# Remove ex_doc plugin (not needed for runtime)
33+
RUN sed -i '/rebar3_ex_doc/d' rebar.config || true
34+
35+
# Build the project
36+
RUN rebar3 compile
37+
38+
# Compile the distributed example module (strip escript header, add module declaration)
39+
RUN (echo '-module(distributed_python).' && sed '1d;/-mode(compile)./d' examples/distributed_python.erl) > /tmp/distributed_python.erl && \
40+
erlc -W0 -o _build/default/lib/erlang_python/ebin /tmp/distributed_python.erl
41+
42+
# Default command - start Erlang with distribution enabled
43+
CMD ["erl", "-pa", "_build/default/lib/erlang_python/ebin", "-eval", "application:ensure_all_started(erlang_python)"]

‎docker/docker-compose.distributed.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Docker Compose for distributed Python execution demo
2+
#
3+
# This creates a 3-node Erlang cluster:
4+
# - coordinator: orchestrates work
5+
# - worker1: Python execution node
6+
# - worker2: Python execution node
7+
#
8+
# Usage:
9+
# docker compose -f docker/docker-compose.distributed.yml up --build
10+
# docker compose -f docker/docker-compose.distributed.yml run coordinator
11+
#
12+
# Then in the coordinator shell:
13+
# distributed_python:demo().
14+
15+
services:
16+
# Coordinator node - runs the demo
17+
coordinator:
18+
build:
19+
context: ..
20+
dockerfile: docker/Dockerfile.distributed
21+
hostname: coordinator
22+
command: ["erl", "-sname", "coordinator", "-setcookie", "distributed_demo", "-pa", "_build/default/lib/erlang_python/ebin", "-eval", "application:ensure_all_started(erlang_python), net_adm:ping(worker1@worker1), net_adm:ping(worker2@worker2)"]
23+
networks:
24+
- erlang_cluster
25+
depends_on:
26+
- worker1
27+
- worker2
28+
stdin_open: true
29+
tty: true
30+
31+
# Worker node 1
32+
worker1:
33+
build:
34+
context: ..
35+
dockerfile: docker/Dockerfile.distributed
36+
hostname: worker1
37+
command: ["erl", "-sname", "worker1", "-setcookie", "distributed_demo", "-pa", "_build/default/lib/erlang_python/ebin", "-eval", "application:ensure_all_started(erlang_python)", "-noshell"]
38+
networks:
39+
- erlang_cluster
40+
41+
# Worker node 2
42+
worker2:
43+
build:
44+
context: ..
45+
dockerfile: docker/Dockerfile.distributed
46+
hostname: worker2
47+
command: ["erl", "-sname", "worker2", "-setcookie", "distributed_demo", "-pa", "_build/default/lib/erlang_python/ebin", "-eval", "application:ensure_all_started(erlang_python)", "-noshell"]
48+
networks:
49+
- erlang_cluster
50+
51+
# Run the demo automatically
52+
demo:
53+
build:
54+
context: ..
55+
dockerfile: docker/Dockerfile.distributed
56+
hostname: demo
57+
command: ["/app/start-distributed-demo.sh"]
58+
networks:
59+
- erlang_cluster
60+
depends_on:
61+
- worker1
62+
- worker2
63+
64+
networks:
65+
erlang_cluster:
66+
driver: bridge

‎docker/run-distributed-demo.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
# Run the distributed Python demo with Docker
3+
#
4+
# This script starts a 3-node Erlang cluster and runs the distributed demo.
5+
#
6+
# Usage:
7+
# ./docker/run-distributed-demo.sh # Run demo automatically
8+
# ./docker/run-distributed-demo.sh shell # Start interactive coordinator shell
9+
# ./docker/run-distributed-demo.sh clean # Clean up containers
10+
11+
set -e
12+
13+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
14+
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
15+
COMPOSE_FILE="$SCRIPT_DIR/docker-compose.distributed.yml"
16+
17+
cd "$PROJECT_DIR"
18+
19+
case "${1:-demo}" in
20+
demo)
21+
echo "Starting distributed Python demo..."
22+
echo "This will start 3 Erlang nodes and run the demo."
23+
echo ""
24+
25+
# Start workers in background
26+
docker compose -f "$COMPOSE_FILE" up -d worker1 worker2
27+
28+
echo "Waiting for workers to start..."
29+
sleep 5
30+
31+
# Run demo
32+
docker compose -f "$COMPOSE_FILE" run --rm demo
33+
34+
# Clean up
35+
docker compose -f "$COMPOSE_FILE" down
36+
;;
37+
38+
shell)
39+
echo "Starting distributed cluster with interactive coordinator..."
40+
echo ""
41+
echo "Once in the shell, run:"
42+
echo " net_adm:ping('worker1@worker1')."
43+
echo " net_adm:ping('worker2@worker2')."
44+
echo " nodes()."
45+
echo " distributed_python:demo()."
46+
echo ""
47+
48+
# Start workers in background
49+
docker compose -f "$COMPOSE_FILE" up -d worker1 worker2
50+
51+
echo "Waiting for workers to start..."
52+
sleep 5
53+
54+
# Start interactive coordinator
55+
docker compose -f "$COMPOSE_FILE" run --rm coordinator
56+
57+
# Clean up
58+
docker compose -f "$COMPOSE_FILE" down
59+
;;
60+
61+
clean)
62+
echo "Cleaning up containers..."
63+
docker compose -f "$COMPOSE_FILE" down --volumes --remove-orphans
64+
;;
65+
66+
logs)
67+
docker compose -f "$COMPOSE_FILE" logs -f
68+
;;
69+
70+
*)
71+
echo "Usage: $0 [demo|shell|clean|logs]"
72+
echo ""
73+
echo "Commands:"
74+
echo " demo - Run the distributed demo automatically (default)"
75+
echo " shell - Start interactive coordinator shell"
76+
echo " clean - Clean up containers"
77+
echo " logs - Show logs from all containers"
78+
exit 1
79+
;;
80+
esac

0 commit comments

Comments
 (0)