"""vllm-router -- single-process front door for the vLLM serving stack. Plan v3.3 section 6. One process, ONE asyncio event loop, TWO listening sockets: * public :8000 -> OpenAI API (/v1/*), /health, (/metrics) * admin :8010 -> /admin/* (for `vllmctl`; published by compose as `127.0.0.1:8010:8010`, so it is host-loopback only. Inside the container it binds 0.0.0.0 because docker cannot publish to a container-loopback bind -- see config.ROUTER_ADMIN_HOST.) Two uvicorn.Server instances are run as coroutines on the same loop because uvicorn cannot bind two ports in one worker, and two *processes* would break the in-process wake locks -- the admin listener must share the very same ServiceManager, asyncio locks and depth tracking as the request path. What it owns: model-name -> service mapping, wake-on-request (single-flight, depth-aware hold), streaming proxy, tiered idle sleep, dev-endpoint hiding (404 for anything outside the allowlist, matched on the *normalized* path). Configuration (environment; authoritative list + defaults in config.py): registry VLLM_TEXT_MODEL=Qwen3.6-35B-A3B-FP8 VLLM_OCR_MODEL=OvisOCR2 VLLM_EMBED_MODEL=Qwen3-Embedding-8B (+ *_SERVICE, *_ALIASES) listeners ROUTER_PUBLIC_PORT=8000 ROUTER_ADMIN_PORT=8010 ROUTER_ADMIN_HOST=127.0.0.1 proxy ROUTER_CONNECT_TIMEOUT=5 ROUTER_READ_TIMEOUT=900 ROUTER_POOL_SIZE=32 ROUTER_MAX_BODY_BYTES=134217728 ROUTER_STATE_CACHE_TTL=2 (keep short: /health returns 200 on a sleeping backend, so readiness is gated on /is_sleeping) wake WAKE_HOLD_SLEEP_S=30 WAKE_HOLD_OFFLOAD_S=180 WAKE_HOLD_RESTART_S=600 (text cold boot measured ~10 min), WAKE_RETRY_AFTER_{SLEEP,OFFLOAD,RESTART}=10/60/600, WAKE_ATTEMPTS=2, WAKE_HTTP_TIMEOUT=300 idle IDLE_SLEEP_MIN=15 (level 1) IDLE_OFFLOAD_MIN=180 (level 2) IDLE_POLL_SECONDS=30 IDLE_ENABLED=1 Run: python app.py (see Dockerfile; `pytest` from router/tests) """ from __future__ import annotations import asyncio import contextlib import logging import signal import sys from pathlib import Path # Sibling modules (config/services/routing/admin_api) -- makes the app work # both as `python app.py` and when imported by the test suite. sys.path.insert(0, str(Path(__file__).resolve().parent)) import uvicorn # noqa: E402 from admin_api import build_admin_app # noqa: E402 from config import Config, load_config # noqa: E402 from routing import build_public_app # noqa: E402 from services import ServiceManager # noqa: E402 log = logging.getLogger("vllm_router") ROUTES = ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS") def setup_logging(level: str = "info") -> None: logging.basicConfig( level=getattr(logging, level.upper(), logging.INFO), format="%(asctime)s %(levelname)-7s %(name)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%S", ) for noisy in ("httpx", "httpcore", "uvicorn.access"): logging.getLogger(noisy).setLevel(logging.WARNING) @contextlib.contextmanager def _no_signal_capture() -> "contextlib.AbstractContextManager[None]": """Replaces uvicorn.Server.capture_signals (see serve()).""" yield def build_apps(cfg: Config | None = None, transport=None): """Build the public + admin apps around one shared ServiceManager. `transport` is an httpx transport override used by the test suite to stub the vLLM backends without any network, GPU or docker. """ cfg = cfg or load_config() manager = ServiceManager(cfg, transport=transport) public_app = build_public_app(manager) admin_app = build_admin_app(manager) @contextlib.asynccontextmanager async def lifespan(_app): manager.start() # idempotent: called by both listeners yield await manager.stop() # idempotent as well # Both apps share one manager; the lifespan is attached post-construction # so there is exactly one definition of it. public_app.router.lifespan_context = lifespan admin_app.router.lifespan_context = lifespan return public_app, admin_app, manager def _make_server(app, host: str, port: int, cfg: Config) -> uvicorn.Server: return uvicorn.Server(uvicorn.Config( app, host=host, port=port, log_level=cfg.log_level, access_log=cfg.access_log, # Long generations must not be cut off, but shutdown must terminate. timeout_graceful_shutdown=10, # The app does its own (structured) request logging. timeout_keep_alive=75, )) async def serve(cfg: Config | None = None) -> None: cfg = cfg or load_config() setup_logging(cfg.log_level) public_app, admin_app, manager = build_apps(cfg) servers = [ _make_server(public_app, cfg.public_host, cfg.public_port, cfg), _make_server(admin_app, cfg.admin_host, cfg.admin_port, cfg), ] # uvicorn's signal capture is per-Server; with two servers the inner # capture would restore its own handler and swallow SIGTERM for the other # listener. Disable it and own the signals here (falling back gracefully # on uvicorn versions without that method). for server in servers: if hasattr(server, "capture_signals"): server.capture_signals = _no_signal_capture loop = asyncio.get_running_loop() stopping = asyncio.Event() def _request_shutdown() -> None: if stopping.is_set(): return stopping.set() log.info("router_stop signal_received") # Flag first, so requests dying in the shutdown window answer with a # depth-aware 503 instead of a bare 500 (E2E case 12). manager.begin_shutdown() for server in servers: server.should_exit = True for sig in (signal.SIGINT, signal.SIGTERM): with contextlib.suppress(NotImplementedError, ValueError, RuntimeError): loop.add_signal_handler(sig, _request_shutdown) async def _propagate_exit() -> None: """Belt and braces: if only one server saw the shutdown, tell the other.""" while not stopping.is_set(): if any(s.should_exit for s in servers): for s in servers: s.should_exit = True await asyncio.sleep(0.1) watchdog = asyncio.create_task(_propagate_exit(), name="exit-watchdog") log.info("router_start public=%s:%d admin=%s:%d services=%s", cfg.public_host, cfg.public_port, cfg.admin_host, cfg.admin_port, ",".join(f"{s.key}={s.service}/{s.model}" for s in cfg.services.values())) try: await asyncio.gather(*(server.serve() for server in servers)) finally: watchdog.cancel() with contextlib.suppress(asyncio.CancelledError): await watchdog await manager.stop() log.info("router_stopped") def main() -> int: try: asyncio.run(serve()) except KeyboardInterrupt: # pragma: no cover - interactive runs return 0 return 0 if __name__ == "__main__": raise SystemExit(main())