Initial commit: router front-door vLLM stack

Three always-running vLLM services (text TP=2 GPU0+1, ocr + embed on GPU2,
sleep mode) behind a FastAPI router that auto-wakes models on request.
Tiered idle (sleep 15 min / offload 3 h), depth-aware 503s with
Retry-After, persisted wake-intent recovery, admin API on 127.0.0.1:8010.
Routine control via vllmctl is pure HTTP — no docker on the request path.

Verified: 91 router unit tests + 15-test E2E on real hardware
(measurements in CALIBRATION.md; design record in
.claude/memory/router-front-door-plan.md).

Old nginx stack files removed before git init; design survives in
.claude/memory/sleep-mode-implementation-plan.md.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-17 10:17:42 +00:00
commit 80eef4ce6a
35 changed files with 6506 additions and 0 deletions

View File

@@ -0,0 +1,120 @@
"""Admin API surface (this is what `vllmctl` talks to)."""
from __future__ import annotations
import time
from config import DEPTH_AWAKE, DEPTH_OFFLOADED
async def test_status_shape(adm, backend):
backend.set_sleeping("ocr", True, level=1)
r = await adm.get("/admin/status")
assert r.status_code == 200
body = r.json()
assert set(body["services"]) == {"text", "ocr", "embed"}
text = body["services"]["text"]
for field in ("service", "model", "base_url", "reachable", "sleeping",
"depth", "wake_in_progress", "active_requests",
"last_activity_ago_s", "last_wake_latency_s", "last_error"):
assert field in text, field
assert text["model"] == "Qwen3.6-35B-A3B-FP8"
assert text["base_url"] == "http://vllm-text:8000"
assert body["services"]["ocr"]["sleeping"] is True
assert body["services"]["ocr"]["depth"] == "offloaded" # unknown level
assert "http://127.0.0.1:8000/v1" in body["api"]
async def test_wake_endpoint(adm, backend):
backend.set_sleeping("embed", True)
r = await adm.post("/admin/wake/embed")
assert r.status_code == 200
body = r.json()
assert body["ok"] is True
assert body["depth"] == "awake"
assert backend.services["vllm-embed"]["wake_up_calls"] == 1
async def test_wake_endpoint_reports_503_when_it_cannot_wake(adm, backend, stack):
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_fails"] = 5
r = await adm.post("/admin/wake/text")
assert r.status_code == 503
assert r.headers["retry-after"] == "60"
body = r.json()
assert body["ok"] is False
assert body["error"]["sleep_depth"] == "offloaded"
assert body["error"]["estimated_wake_seconds"] == 60
async def test_wake_accepts_model_name_and_case(adm, backend):
backend.set_sleeping("ocr", True)
r = await adm.post("/admin/wake/OvisOCR2")
assert r.status_code == 200
assert backend.services["vllm-ocr"]["wake_up_calls"] == 1
async def test_wake_unknown_key_is_404(adm):
assert (await adm.post("/admin/wake/nope")).status_code == 404
async def test_sleep_level1_and_level2(adm, backend, stack):
r = await adm.post("/admin/sleep/text?level=1")
assert r.status_code == 200
assert r.json()["ok"] is True
assert "level=1" in backend.calls[backend.last("POST vllm-text/sleep")]
assert stack.manager.services["text"].depth == "sleeping"
r = await adm.post("/admin/sleep/text?level=2")
assert r.status_code == 200
assert "level=2" in backend.calls[backend.last("POST vllm-text/sleep")]
assert stack.manager.services["text"].depth == DEPTH_OFFLOADED
async def test_sleep_defaults_to_level1(adm, backend):
r = await adm.post("/admin/sleep/embed")
assert r.status_code == 200
assert "level=1" in backend.calls[backend.last("POST vllm-embed/sleep")]
async def test_sleep_rejects_bad_level(adm):
assert (await adm.post("/admin/sleep/text?level=3")).status_code == 400
assert (await adm.post("/admin/sleep/text?level=abc")).status_code == 400
async def test_sleep_unknown_key_is_404(adm, backend):
assert (await adm.post("/admin/sleep/nope")).status_code == 404
assert backend.calls == []
async def test_sleep_idempotent_when_already_at_depth(adm, backend, stack):
await adm.post("/admin/sleep/ocr?level=1")
backend.calls.clear()
r = await adm.post("/admin/sleep/ocr?level=1")
assert r.status_code == 200
assert r.json()["already"] is True
assert backend.count("POST vllm-ocr/sleep") == 0 # no second POST /sleep
async def test_admin_wake_resets_the_idle_clock(adm, backend, stack):
backend.set_sleeping("text", True)
text = stack.manager.services["text"]
text.last_activity -= 10_000
before = time.monotonic() - text.last_activity
assert (await adm.post("/admin/wake/text")).status_code == 200
assert text.depth == DEPTH_AWAKE
assert (time.monotonic() - text.last_activity) < before
async def test_admin_health(adm):
r = await adm.get("/health")
assert r.status_code == 200
assert r.json()["status"] == "ok"
async def test_admin_health_alias(adm):
"""The documented admin surface says /admin/health."""
assert (await adm.get("/admin/health")).status_code == 200
body = (await adm.get("/admin/health")).json()
assert body["status"] == "ok"
assert "awake" in body