Feature: separate API process (#393)

Problem: Users experience latency issues with the CCN API.
This is caused by some parts of the code being synchronous:
inherently for filesystem accesses, by design for DB accesses.

Solution: run multiple API processes.

We now use gunicorn to spawn several API worker processes.
The API is now spawned as an independent container to make it
easier to configure.

The API port and number of workers can be configured by
setting the following environment variables:

- `CCN_CONFIG_API_PORT`
- `CCN_CONFIG_API_NB_WORKERS`.

Author: odesenfans

deployment/samples/docker-compose/docker-compose.yml

@@ -8,16 +8,12 @@ volumes:
 services:
   pyaleph:
     restart: always
-    image: alephim/pyaleph-node:v0.5.0-rc1
+    image: alephim/pyaleph-node:latest
     command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
-    ports:
-      - "127.0.0.1:8000:8000/tcp"
-      - "4024:4024/tcp"
     volumes:
       - ./config.yml:/opt/pyaleph/config.yml
       - ./keys:/opt/pyaleph/keys
       - pyaleph-local-storage:/var/lib/pyaleph
-
     depends_on:
       - postgres
       - ipfs
@@ -29,6 +25,27 @@ services:
       options:
         max-size: 50m
 
+  pyaleph-api:
+    restart: always
+    image: alephim/pyaleph-node:latest
+    command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
+    entrypoint: ["bash", "deployment/scripts/run_aleph_ccn_api.sh"]
+    ports:
+      - "4024:4024/tcp"
+    volumes:
+      - ./config.yml:/opt/pyaleph/config.yml
+      - pyaleph-local-storage:/var/lib/pyaleph
+    environment:
+      CCN_CONFIG_API_PORT: 4024
+      CCN_CONFIG_API_NB_WORKERS: 8
+    depends_on:
+      - pyaleph
+    networks:
+      - pyaleph
+    logging:
+      options:
+        max-size: 50m
+
   p2p-service:
     restart: always
     image: alephim/p2p-service:0.1.2

deployment/samples/docker-monitoring/docker-compose.yml

@@ -10,11 +10,8 @@ volumes:
 services:
   pyaleph:
     restart: always
-    image: alephim/pyaleph-node:v0.5.0-rc1
+    image: alephim/pyaleph-node:latest
     command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
-    ports:
-      - "127.0.0.1:8000:8000/tcp"
-      - "4024:4024/tcp"
     volumes:
       - ./config.yml:/opt/pyaleph/config.yml
       - ./keys:/opt/pyaleph/keys
@@ -30,6 +27,27 @@ services:
       options:
         max-size: 50m
 
+  pyaleph-api:
+    restart: always
+    image: alephim/pyaleph-node:latest
+    command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
+    entrypoint: ["bash", "deployment/scripts/run_aleph_ccn_api.sh"]
+    ports:
+      - "4024:4024/tcp"
+    volumes:
+      - ./config.yml:/opt/pyaleph/config.yml
+      - pyaleph-local-storage:/var/lib/pyaleph
+    environment:
+      CCN_CONFIG_API_PORT: 4024
+      CCN_CONFIG_API_NB_WORKERS: 8
+    depends_on:
+      - pyaleph
+    networks:
+      - pyaleph
+    logging:
+      options:
+        max-size: 50m
+
   p2p-service:
     restart: always
     image: alephim/p2p-service:0.1.2
@@ -106,7 +124,7 @@ services:
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     networks:
-      - pyaleph
+      - pyaleph-api
       - grafana
 
   grafana:

deployment/scripts/run_aleph_ccn.sh

@@ -26,36 +26,7 @@ while test $# -gt 0; do
   shift
 done
 
-function get_config()
-{
-  config_key="$1"
-  config_value=$(python3 "${SCRIPT_DIR}/get_config_value.py" --config-file "${CONFIG_FILE}" "${config_key}")
-  echo "${config_value}"
-}
-
-function wait_for_it()
-{
-  "${SCRIPT_DIR}"/wait-for-it.sh "$@"
-}
-
-POSTGRES_HOST=$(get_config postgres.host)
-POSTGRES_PORT=$(get_config postgres.port)
-IPFS_HOST=$(get_config ipfs.host)
-IPFS_PORT=$(get_config ipfs.port)
-RABBITMQ_HOST=$(get_config rabbitmq.host)
-RABBITMQ_PORT=$(get_config rabbitmq.port)
-REDIS_HOST=$(get_config redis.host)
-REDIS_PORT=$(get_config redis.port)
-P2P_SERVICE_HOST=$(get_config p2p.daemon_host)
-P2P_SERVICE_CONTROL_PORT=$(get_config p2p.control_port)
-
-if [ "$(get_config ipfs.enabled)" = "True" ]; then
-  wait_for_it -h "${IPFS_HOST}" -p "${IPFS_PORT}"
-fi
-
-wait_for_it -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}"
-wait_for_it -h "${RABBITMQ_HOST}" -p "${RABBITMQ_PORT}"
-wait_for_it -h "${REDIS_HOST}" -p "${REDIS_PORT}"
-wait_for_it -h "${P2P_SERVICE_HOST}" -p "${P2P_SERVICE_CONTROL_PORT}"
+source ${SCRIPT_DIR}/wait_for_services.sh
+wait_for_services "${CONFIG_FILE}"
 
 exec pyaleph "${PYALEPH_ARGS[@]}"

deployment/scripts/run_aleph_ccn_api.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Starts an Aleph Core Channel Node API server.
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CONFIG_FILE="/var/pyaleph/config.yml"
+
+while test $# -gt 0; do
+  case "$1" in
+  --help)
+    help
+    ;;
+  --config)
+    CONFIG_FILE="$2"
+    shift
+    ;;
+  esac
+  shift
+done
+
+source ${SCRIPT_DIR}/wait_for_services.sh
+wait_for_services "${CONFIG_FILE}"
+
+NB_WORKERS="${CCN_CONFIG_API_NB_WORKERS:-4}"
+PORT=${CCN_CONFIG_API_PORT:-4024}
+
+echo "Starting aleph.im CCN API server on port ${PORT} (${NB_WORKERS} workers)"
+
+exec gunicorn \
+  "aleph.api_entrypoint:create_app" \
+  --bind 0.0.0.0:${PORT} \
+  --worker-class aiohttp.worker.GunicornUVLoopWebWorker \
+  --workers ${NB_WORKERS} \
+  --access-logfile "-"

deployment/scripts/wait_for_services.sh

@@ -0,0 +1,36 @@
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+function get_config() {
+  config_file="$1"
+  config_key="$2"
+  config_value=$(python3 "${SCRIPT_DIR}/get_config_value.py" --config-file "${CONFIG_FILE}" "${config_key}")
+  echo "${config_value}"
+}
+
+function wait_for_it() {
+  "${SCRIPT_DIR}"/wait-for-it.sh "$@"
+}
+
+function wait_for_services() {
+  config_file="$1"
+
+  POSTGRES_HOST=$(get_config "${config_file}" postgres.host)
+  POSTGRES_PORT=$(get_config "${config_file}" postgres.port)
+  IPFS_HOST=$(get_config "${config_file}" ipfs.host)
+  IPFS_PORT=$(get_config "${config_file}" ipfs.port)
+  RABBITMQ_HOST=$(get_config "${config_file}" rabbitmq.host)
+  RABBITMQ_PORT=$(get_config "${config_file}" rabbitmq.port)
+  REDIS_HOST=$(get_config "${config_file}" redis.host)
+  REDIS_PORT=$(get_config "${config_file}" redis.port)
+  P2P_SERVICE_HOST=$(get_config "${config_file}" p2p.daemon_host)
+  P2P_SERVICE_CONTROL_PORT=$(get_config "${config_file}" p2p.control_port)
+
+  if [ "$(get_config "${config_file}" ipfs.enabled)" = "True" ]; then
+    wait_for_it -h "${IPFS_HOST}" -p "${IPFS_PORT}"
+  fi
+
+  wait_for_it -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}"
+  wait_for_it -h "${RABBITMQ_HOST}" -p "${RABBITMQ_PORT}"
+  wait_for_it -h "${REDIS_HOST}" -p "${REDIS_PORT}"
+  wait_for_it -h "${P2P_SERVICE_HOST}" -p "${P2P_SERVICE_CONTROL_PORT}"
+}

setup.cfg

@@ -50,6 +50,7 @@ install_requires =
     cosmospy==6.0.0
     dataclasses_json==0.5.6
     eth_account==0.8.0
+    gunicorn==20.1.0
     hexbytes==0.2.2
     msgpack==1.0.3  # required by aiocache
     multiaddr==0.0.9    # for libp2p-stubs
@@ -72,6 +73,7 @@ install_requires =
     substrate-interface==1.3.4
     ujson==5.1.0  # required by aiocache
     urllib3==1.26.11
+    uvloop==0.17.0
     web3==6.0.0b9
 
 dependency_links =

src/aleph/api_entrypoint.py

@@ -0,0 +1,78 @@
+import logging
+from pathlib import Path
+
+import sentry_sdk
+from aiohttp import web
+from configmanager import Config
+
+import aleph.config
+from aleph.db.connection import make_engine, make_session_factory
+from aleph.services.cache.node_cache import NodeCache
+from aleph.services.ipfs import IpfsService
+from aleph.services.ipfs.common import make_ipfs_client
+from aleph.services.p2p import init_p2p_client
+from aleph.services.storage.fileystem_engine import FileSystemStorageEngine
+from aleph.storage import StorageService
+from aleph.toolkit.monitoring import setup_sentry
+from aleph.web import create_aiohttp_app
+from aleph.web.controllers.app_state_getters import (
+    APP_STATE_CONFIG,
+    APP_STATE_MQ_CONN,
+    APP_STATE_NODE_CACHE,
+    APP_STATE_P2P_CLIENT,
+    APP_STATE_SESSION_FACTORY,
+    APP_STATE_STORAGE_SERVICE,
+)
+
+
+async def configure_aiohttp_app(
+    config: Config,
+) -> web.Application:
+    with sentry_sdk.start_transaction(name=f"init-api-server"):
+        p2p_client = await init_p2p_client(config, service_name=f"api-server-aiohttp")
+
+        engine = make_engine(
+            config,
+            echo=config.logging.level.value == logging.DEBUG,
+            application_name=f"aleph-api",
+        )
+        session_factory = make_session_factory(engine)
+
+        node_cache = NodeCache(
+            redis_host=config.redis.host.value, redis_port=config.redis.port.value
+        )
+
+        ipfs_client = make_ipfs_client(config)
+        ipfs_service = IpfsService(ipfs_client=ipfs_client)
+        storage_service = StorageService(
+            storage_engine=FileSystemStorageEngine(folder=config.storage.folder.value),
+            ipfs_service=ipfs_service,
+            node_cache=node_cache,
+        )
+
+        app = create_aiohttp_app()
+
+        app[APP_STATE_CONFIG] = config
+        app[APP_STATE_P2P_CLIENT] = p2p_client
+        # Reuse the connection of the P2P client to avoid opening two connections
+        app[APP_STATE_MQ_CONN] = p2p_client.mq_client.connection
+        app[APP_STATE_NODE_CACHE] = node_cache
+        app[APP_STATE_STORAGE_SERVICE] = storage_service
+        app[APP_STATE_SESSION_FACTORY] = session_factory
+
+    return app
+
+
+async def create_app() -> web.Application:
+    config = aleph.config.app_config
+
+    # TODO: make the config file path configurable
+    config_file = Path.cwd() / "config.yml"
+    config.yaml.load(str(config_file))
+
+    logging.basicConfig(level=config.logging.level.value)
+
+    if config.sentry.dsn.value:
+        setup_sentry(config)
+
+    return await configure_aiohttp_app(config=config)