vllm-frontdoor/compose.yml

# vLLM serving stack — router front door (plan v3.3, 2026-08-17).
#
# nginx is GONE: the FastAPI router (./router) is the only public ingress
# (host :8000), with a separate admin listener on 127.0.0.1:8010.
# Each model is a dedicated always-running vLLM service with Sleep Mode
# enabled; the router wakes them on request.
#
# Hardware constraints (NOTES-2026-08-13.md, NOTES-2026-08-17-gpu2-recheck.md):
#   * GPU P2P DMA corrupts data host-wide (4/6 paths). NCCL_P2P_DISABLE=1
#     + --disable-custom-all-reduce are MANDATORY on every service.
#   * text runs TP=2 on GPU0+GPU1 (exclusive); OCR and embed run TP=1 on
#     GPU2 (single-GPU services never issue P2P transfers).
#   * Moving a service to another GPU = edit its CUDA_VISIBLE_DEVICES (then
#     re-tighten slices).
#
# No healthchecks on purpose: sleep mode makes /health flap, and nothing here
# may use a service_healthy dependency (plan §7).

services:
  vllm-text:
    image: vllm/vllm-openai:${VLLM_VERSION:-v0.27.1}
    container_name: vllm-text
    restart: unless-stopped
    ports:
      - "127.0.0.1:8001:8000" # direct debug access (local only)
    volumes:
      - ${MODEL_ROOT:-/data/home/renbaibing/huggingface}:/models:ro
    ipc: host
    gpus: all
    environment:
      VLLM_SERVER_DEV_MODE: "1" # sleep-mode endpoints (/sleep /wake_up /is_sleeping ...)
      HF_TOKEN: ${HF_TOKEN:-}
      NCCL_P2P_DISABLE: "1" # broken P2P DMA on this host (see header)
      CUDA_VISIBLE_DEVICES: "0,1"
    command: >
      /models/Qwen3.6-35B-A3B-FP8
      --served-model-name Qwen3.6-35B-A3B-FP8
      --tensor-parallel-size 2
      --max-model-len 262144
      --gpu-memory-utilization 0.85
      --reasoning-parser qwen3
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --disable-custom-all-reduce
      --enable-sleep-mode

  vllm-ocr:
    image: vllm/vllm-openai:${VLLM_VERSION:-v0.27.1}
    container_name: vllm-ocr
    restart: unless-stopped
    ports:
      - "127.0.0.1:8002:8000" # direct debug access (local only)
    volumes:
      - ${MODEL_ROOT:-/data/home/renbaibing/huggingface}:/models:ro
    ipc: host
    gpus: all
    environment:
      VLLM_SERVER_DEV_MODE: "1"
      HF_TOKEN: ${HF_TOKEN:-}
      NCCL_P2P_DISABLE: "1"
      CUDA_VISIBLE_DEVICES: "2"
    command: >
      /models/OvisOCR2
      --served-model-name OvisOCR2
      --tensor-parallel-size 1
      --max-model-len 32768
      --gpu-memory-utilization 0.10
      --max-num-seqs 256
      --disable-custom-all-reduce
      --enable-sleep-mode

  vllm-embed:
    image: vllm/vllm-openai:${VLLM_VERSION:-v0.27.1}
    container_name: vllm-embed
    restart: unless-stopped
    ports:
      - "127.0.0.1:8003:8000" # direct debug access (local only)
    volumes:
      - ${MODEL_ROOT:-/data/home/renbaibing/huggingface}:/models:ro
    ipc: host
    gpus: all
    environment:
      VLLM_SERVER_DEV_MODE: "1"
      HF_TOKEN: ${HF_TOKEN:-}
      NCCL_P2P_DISABLE: "1"
      CUDA_VISIBLE_DEVICES: "2"
    command: >
      /models/Qwen3-Embedding-8B
      --served-model-name Qwen3-Embedding-8B
      --tensor-parallel-size 1
      --max-model-len 8192
      --gpu-memory-utilization 0.25
      --disable-custom-all-reduce
      --enable-sleep-mode

  router:
    build: ./router
    container_name: vllm-router
    restart: unless-stopped
    ports:
      - "8000:8000" # public API — only public ingress
      - "127.0.0.1:8010:8010" # admin API (vllmctl / debugging), local only
    volumes:
      # Wake-intent state: lets a restarted router tell "awake and ready" from
      # "awake because the previous one died between wake_up and reload_weights"
      # (plan 6.4 / E2E case 12). Written atomically, a few hundred bytes.
      # Named volume (not a bind mount): Docker seeds it with the image's
      # /state ownership, so the non-root router user can always write it.
      - router-state:/state
    depends_on:
      vllm-text:
        condition: service_started
      vllm-ocr:
        condition: service_started
      vllm-embed:
        condition: service_started

volumes:
  router-state: