bing
80eef4ce6a
Three always-running vLLM services (text TP=2 GPU0+1, ocr + embed on GPU2, sleep mode) behind a FastAPI router that auto-wakes models on request. Tiered idle (sleep 15 min / offload 3 h), depth-aware 503s with Retry-After, persisted wake-intent recovery, admin API on 127.0.0.1:8010. Routine control via vllmctl is pure HTTP — no docker on the request path. Verified: 91 router unit tests + 15-test E2E on real hardware (measurements in CALIBRATION.md; design record in .claude/memory/router-front-door-plan.md). Old nginx stack files removed before git init; design survives in .claude/memory/sleep-mode-implementation-plan.md. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
121 lines
4.3 KiB
Python
121 lines
4.3 KiB
Python
"""Admin API surface (this is what `vllmctl` talks to)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
|
|
from config import DEPTH_AWAKE, DEPTH_OFFLOADED
|
|
|
|
|
|
async def test_status_shape(adm, backend):
|
|
backend.set_sleeping("ocr", True, level=1)
|
|
r = await adm.get("/admin/status")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert set(body["services"]) == {"text", "ocr", "embed"}
|
|
text = body["services"]["text"]
|
|
for field in ("service", "model", "base_url", "reachable", "sleeping",
|
|
"depth", "wake_in_progress", "active_requests",
|
|
"last_activity_ago_s", "last_wake_latency_s", "last_error"):
|
|
assert field in text, field
|
|
assert text["model"] == "Qwen3.6-35B-A3B-FP8"
|
|
assert text["base_url"] == "http://vllm-text:8000"
|
|
assert body["services"]["ocr"]["sleeping"] is True
|
|
assert body["services"]["ocr"]["depth"] == "offloaded" # unknown level
|
|
assert "http://127.0.0.1:8000/v1" in body["api"]
|
|
|
|
|
|
async def test_wake_endpoint(adm, backend):
|
|
backend.set_sleeping("embed", True)
|
|
r = await adm.post("/admin/wake/embed")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["ok"] is True
|
|
assert body["depth"] == "awake"
|
|
assert backend.services["vllm-embed"]["wake_up_calls"] == 1
|
|
|
|
|
|
async def test_wake_endpoint_reports_503_when_it_cannot_wake(adm, backend, stack):
|
|
backend.set_sleeping("text", True)
|
|
backend.services["vllm-text"]["wake_fails"] = 5
|
|
r = await adm.post("/admin/wake/text")
|
|
assert r.status_code == 503
|
|
assert r.headers["retry-after"] == "60"
|
|
body = r.json()
|
|
assert body["ok"] is False
|
|
assert body["error"]["sleep_depth"] == "offloaded"
|
|
assert body["error"]["estimated_wake_seconds"] == 60
|
|
|
|
|
|
async def test_wake_accepts_model_name_and_case(adm, backend):
|
|
backend.set_sleeping("ocr", True)
|
|
r = await adm.post("/admin/wake/OvisOCR2")
|
|
assert r.status_code == 200
|
|
assert backend.services["vllm-ocr"]["wake_up_calls"] == 1
|
|
|
|
|
|
async def test_wake_unknown_key_is_404(adm):
|
|
assert (await adm.post("/admin/wake/nope")).status_code == 404
|
|
|
|
|
|
async def test_sleep_level1_and_level2(adm, backend, stack):
|
|
r = await adm.post("/admin/sleep/text?level=1")
|
|
assert r.status_code == 200
|
|
assert r.json()["ok"] is True
|
|
assert "level=1" in backend.calls[backend.last("POST vllm-text/sleep")]
|
|
assert stack.manager.services["text"].depth == "sleeping"
|
|
|
|
r = await adm.post("/admin/sleep/text?level=2")
|
|
assert r.status_code == 200
|
|
assert "level=2" in backend.calls[backend.last("POST vllm-text/sleep")]
|
|
assert stack.manager.services["text"].depth == DEPTH_OFFLOADED
|
|
|
|
|
|
async def test_sleep_defaults_to_level1(adm, backend):
|
|
r = await adm.post("/admin/sleep/embed")
|
|
assert r.status_code == 200
|
|
assert "level=1" in backend.calls[backend.last("POST vllm-embed/sleep")]
|
|
|
|
|
|
async def test_sleep_rejects_bad_level(adm):
|
|
assert (await adm.post("/admin/sleep/text?level=3")).status_code == 400
|
|
assert (await adm.post("/admin/sleep/text?level=abc")).status_code == 400
|
|
|
|
|
|
async def test_sleep_unknown_key_is_404(adm, backend):
|
|
assert (await adm.post("/admin/sleep/nope")).status_code == 404
|
|
assert backend.calls == []
|
|
|
|
|
|
async def test_sleep_idempotent_when_already_at_depth(adm, backend, stack):
|
|
await adm.post("/admin/sleep/ocr?level=1")
|
|
backend.calls.clear()
|
|
r = await adm.post("/admin/sleep/ocr?level=1")
|
|
assert r.status_code == 200
|
|
assert r.json()["already"] is True
|
|
assert backend.count("POST vllm-ocr/sleep") == 0 # no second POST /sleep
|
|
|
|
|
|
async def test_admin_wake_resets_the_idle_clock(adm, backend, stack):
|
|
backend.set_sleeping("text", True)
|
|
text = stack.manager.services["text"]
|
|
text.last_activity -= 10_000
|
|
before = time.monotonic() - text.last_activity
|
|
assert (await adm.post("/admin/wake/text")).status_code == 200
|
|
assert text.depth == DEPTH_AWAKE
|
|
assert (time.monotonic() - text.last_activity) < before
|
|
|
|
|
|
async def test_admin_health(adm):
|
|
r = await adm.get("/health")
|
|
assert r.status_code == 200
|
|
assert r.json()["status"] == "ok"
|
|
|
|
|
|
async def test_admin_health_alias(adm):
|
|
"""The documented admin surface says /admin/health."""
|
|
assert (await adm.get("/admin/health")).status_code == 200
|
|
body = (await adm.get("/admin/health")).json()
|
|
assert body["status"] == "ok"
|
|
assert "awake" in body
|