vllm-frontdoor/router/app.py

"""vllm-router -- single-process front door for the vLLM serving stack.

Plan v3.3 section 6.  One process, ONE asyncio event loop, TWO listening
sockets:

    * public  :8000  -> OpenAI API (/v1/*), /health, (/metrics)
    * admin   :8010  -> /admin/* (for `vllmctl`; published by compose as
      `127.0.0.1:8010:8010`, so it is host-loopback only.  Inside the
      container it binds 0.0.0.0 because docker cannot publish to a
      container-loopback bind -- see config.ROUTER_ADMIN_HOST.)

Two uvicorn.Server instances are run as coroutines on the same loop because
uvicorn cannot bind two ports in one worker, and two *processes* would break
the in-process wake locks -- the admin listener must share the very same
ServiceManager, asyncio locks and depth tracking as the request path.

What it owns: model-name -> service mapping, wake-on-request (single-flight,
depth-aware hold), streaming proxy, tiered idle sleep, dev-endpoint hiding
(404 for anything outside the allowlist, matched on the *normalized* path).

Configuration (environment; authoritative list + defaults in config.py):

  registry      VLLM_TEXT_MODEL=Qwen3.6-35B-A3B-FP8  VLLM_OCR_MODEL=OvisOCR2
                VLLM_EMBED_MODEL=Qwen3-Embedding-8B  (+ *_SERVICE, *_ALIASES)
  listeners     ROUTER_PUBLIC_PORT=8000  ROUTER_ADMIN_PORT=8010
                ROUTER_ADMIN_HOST=127.0.0.1
  proxy         ROUTER_CONNECT_TIMEOUT=5  ROUTER_READ_TIMEOUT=900
                ROUTER_POOL_SIZE=32  ROUTER_MAX_BODY_BYTES=134217728
                ROUTER_STATE_CACHE_TTL=2 (keep short: /health returns 200 on a
                sleeping backend, so readiness is gated on /is_sleeping)
  wake          WAKE_HOLD_SLEEP_S=30  WAKE_HOLD_OFFLOAD_S=180
                WAKE_HOLD_RESTART_S=600 (text cold boot measured ~10 min),
                WAKE_RETRY_AFTER_{SLEEP,OFFLOAD,RESTART}=10/60/600,
                WAKE_ATTEMPTS=2, WAKE_HTTP_TIMEOUT=300
  idle          IDLE_SLEEP_MIN=15 (level 1)  IDLE_OFFLOAD_MIN=180 (level 2)
                IDLE_POLL_SECONDS=30  IDLE_ENABLED=1

Run:  python app.py      (see Dockerfile; `pytest` from router/tests)
"""

from __future__ import annotations

import asyncio
import contextlib
import logging
import signal
import sys
from pathlib import Path

# Sibling modules (config/services/routing/admin_api) -- makes the app work
# both as `python app.py` and when imported by the test suite.
sys.path.insert(0, str(Path(__file__).resolve().parent))

import uvicorn  # noqa: E402

from admin_api import build_admin_app  # noqa: E402
from config import Config, load_config  # noqa: E402
from routing import build_public_app  # noqa: E402
from services import ServiceManager  # noqa: E402

log = logging.getLogger("vllm_router")

ROUTES = ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS")


def setup_logging(level: str = "info") -> None:
    logging.basicConfig(
        level=getattr(logging, level.upper(), logging.INFO),
        format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
        datefmt="%Y-%m-%dT%H:%M:%S",
    )
    for noisy in ("httpx", "httpcore", "uvicorn.access"):
        logging.getLogger(noisy).setLevel(logging.WARNING)


@contextlib.contextmanager
def _no_signal_capture() -> "contextlib.AbstractContextManager[None]":
    """Replaces uvicorn.Server.capture_signals (see serve())."""
    yield


def build_apps(cfg: Config | None = None, transport=None):
    """Build the public + admin apps around one shared ServiceManager.

    `transport` is an httpx transport override used by the test suite to stub
    the vLLM backends without any network, GPU or docker.
    """
    cfg = cfg or load_config()
    manager = ServiceManager(cfg, transport=transport)
    public_app = build_public_app(manager)
    admin_app = build_admin_app(manager)

    @contextlib.asynccontextmanager
    async def lifespan(_app):
        manager.start()          # idempotent: called by both listeners
        yield
        await manager.stop()     # idempotent as well

    # Both apps share one manager; the lifespan is attached post-construction
    # so there is exactly one definition of it.
    public_app.router.lifespan_context = lifespan
    admin_app.router.lifespan_context = lifespan
    return public_app, admin_app, manager


def _make_server(app, host: str, port: int, cfg: Config) -> uvicorn.Server:
    return uvicorn.Server(uvicorn.Config(
        app,
        host=host,
        port=port,
        log_level=cfg.log_level,
        access_log=cfg.access_log,
        # Long generations must not be cut off, but shutdown must terminate.
        timeout_graceful_shutdown=10,
        # The app does its own (structured) request logging.
        timeout_keep_alive=75,
    ))


async def serve(cfg: Config | None = None) -> None:
    cfg = cfg or load_config()
    setup_logging(cfg.log_level)
    public_app, admin_app, manager = build_apps(cfg)

    servers = [
        _make_server(public_app, cfg.public_host, cfg.public_port, cfg),
        _make_server(admin_app, cfg.admin_host, cfg.admin_port, cfg),
    ]

    # uvicorn's signal capture is per-Server; with two servers the inner
    # capture would restore its own handler and swallow SIGTERM for the other
    # listener.  Disable it and own the signals here (falling back gracefully
    # on uvicorn versions without that method).
    for server in servers:
        if hasattr(server, "capture_signals"):
            server.capture_signals = _no_signal_capture

    loop = asyncio.get_running_loop()
    stopping = asyncio.Event()

    def _request_shutdown() -> None:
        if stopping.is_set():
            return
        stopping.set()
        log.info("router_stop signal_received")
        # Flag first, so requests dying in the shutdown window answer with a
        # depth-aware 503 instead of a bare 500 (E2E case 12).
        manager.begin_shutdown()
        for server in servers:
            server.should_exit = True

    for sig in (signal.SIGINT, signal.SIGTERM):
        with contextlib.suppress(NotImplementedError, ValueError, RuntimeError):
            loop.add_signal_handler(sig, _request_shutdown)

    async def _propagate_exit() -> None:
        """Belt and braces: if only one server saw the shutdown, tell the other."""
        while not stopping.is_set():
            if any(s.should_exit for s in servers):
                for s in servers:
                    s.should_exit = True
            await asyncio.sleep(0.1)

    watchdog = asyncio.create_task(_propagate_exit(), name="exit-watchdog")
    log.info("router_start public=%s:%d admin=%s:%d services=%s",
             cfg.public_host, cfg.public_port, cfg.admin_host, cfg.admin_port,
             ",".join(f"{s.key}={s.service}/{s.model}" for s in cfg.services.values()))
    try:
        await asyncio.gather(*(server.serve() for server in servers))
    finally:
        watchdog.cancel()
        with contextlib.suppress(asyncio.CancelledError):
            await watchdog
        await manager.stop()
        log.info("router_stopped")


def main() -> int:
    try:
        asyncio.run(serve())
    except KeyboardInterrupt:  # pragma: no cover - interactive runs
        return 0
    return 0


if __name__ == "__main__":
    raise SystemExit(main())