composte-hermes.yml aktualisiert
This commit is contained in:
parent
48e4054c2e
commit
6dae3f11e7
|
|
@ -1,10 +1,7 @@
|
|||
# docker-compose.yml – Hermes Agent + vLLM (Qwen3-14B-AWQ)
|
||||
# Host: Cortex (eigenständiger Server, 2x RTX 3060 12GB)
|
||||
#
|
||||
# Beide Services laufen im selben Docker-Netz "cortex-net" auf diesem Host.
|
||||
# Hermes spricht vLLM intern über http://vllm:8000/v1 an – kein extra
|
||||
# Netzwerk-Routing nötig, alles lokal auf Cortex.
|
||||
#
|
||||
# Host: Cortex (2x RTX 3060 12GB)
|
||||
# Daten liegen unter /opt/vllm und /opt/hermes (Sicherung/Backup)
|
||||
|
||||
services:
|
||||
vllm:
|
||||
image: vllm/vllm-openai:latest
|
||||
|
|
@ -16,11 +13,10 @@ services:
|
|||
- "8005:8000"
|
||||
environment:
|
||||
- NVIDIA_VISIBLE_DEVICES=0,1
|
||||
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
||||
# Reduziert Fragmentierung im CUDA-Allocator – hilft bei knappem VRAM
|
||||
- HUGGING_FACE_HUB_TOKEN=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
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- ${HOME:-/root}/.cache/huggingface:/root/.cache/huggingface
|
||||
- /opt/vllm/huggingface:/root/.cache/huggingface
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
|
|
@ -32,25 +28,13 @@ services:
|
|||
- --model=Qwen/Qwen3-14B-AWQ
|
||||
- --quantization=awq
|
||||
- --tensor-parallel-size=2
|
||||
# Von 0.90 auf 0.82 gesenkt: lässt Platz für CUDA-Graph-Pools
|
||||
# und NCCL all_reduce Buffer, die zusätzlich zum KV-Cache-Pool anfallen
|
||||
- --gpu-memory-utilization=0.82
|
||||
# 32768 ist die native Context-Länge von Qwen3-14B – YaRN entfernt,
|
||||
# da hier gar keine Erweiterung über die native Länge hinaus stattfindet
|
||||
- --max-model-len=32768
|
||||
# Begrenzt parallele Sequenzen -> kleinerer KV-Cache-Bedarf,
|
||||
# wichtig bei nur 12GB pro GPU. Bei Bedarf hochsetzen, wenn stabil.
|
||||
- --max-num-seqs=16
|
||||
# FP8 KV-Cache spart ca. 50% Speicher gegenüber FP16 bei minimalem
|
||||
# Qualitätsverlust – gerade bei 32k Context relevant
|
||||
- --kv-cache-dtype=fp8
|
||||
- --served-model-name=qwen3-14b-awq
|
||||
# Fehlten bisher: ohne diese beiden Flags lehnt vLLM jeden Request
|
||||
# mit tool_choice="auto" ab ("auto tool choice requires ...")
|
||||
- --enable-auto-tool-choice
|
||||
- --tool-call-parser=hermes
|
||||
# Trennt <think>-Reasoning-Content vom eigentlichen Tool-Call-Parsing,
|
||||
# sonst kann der hermes-Parser bei aktiviertem Denk-Modus stolpern
|
||||
- --reasoning-parser=deepseek_r1
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
|
|
@ -60,47 +44,44 @@ services:
|
|||
start_period: 180s
|
||||
networks:
|
||||
- cortex-net
|
||||
hermes-gateway:
|
||||
|
||||
hermes-agent:
|
||||
image: nousresearch/hermes-agent:latest
|
||||
container_name: cortex-hermes-gateway
|
||||
container_name: cortex-hermes-agent
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
vllm:
|
||||
condition: service_healthy
|
||||
command: ["gateway", "run"]
|
||||
shm_size: 1g
|
||||
ports:
|
||||
- "8642:8642"
|
||||
- "9119:9119" # Dashboard, jetzt im LAN erreichbar
|
||||
- "8642:8642" # Gateway-API, im LAN erreichbar
|
||||
volumes:
|
||||
- ${HOME:-/root}/.hermes:/opt/data
|
||||
- /opt/hermes:/opt/data
|
||||
environment:
|
||||
- HERMES_UID=${HERMES_UID:-10000}
|
||||
- HERMES_GID=${HERMES_GID:-10000}
|
||||
# Zeigt Hermes auf den lokalen vLLM-Server
|
||||
- OPENAI_BASE_URL=http://vllm:8000/v1
|
||||
- OPENAI_API_KEY=dummy
|
||||
# Gateway-API nur aktivieren, wenn du sie extern brauchst:
|
||||
# - API_SERVER_HOST=0.0.0.0
|
||||
# - API_SERVER_KEY=${API_SERVER_KEY}
|
||||
networks:
|
||||
- cortex-net
|
||||
hermes-dashboard:
|
||||
image: nousresearch/hermes-agent:latest
|
||||
container_name: cortex-hermes-dashboard
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- hermes-gateway
|
||||
command: ["dashboard", "--host", "0.0.0.0", "--no-open"]
|
||||
ports:
|
||||
- "127.0.0.1:9119:9119" # nur lokal auf Cortex, kein LAN-Expose
|
||||
volumes:
|
||||
- ${HOME:-/root}/.hermes:/opt/data
|
||||
environment:
|
||||
- HERMES_UID=${HERMES_UID:-10000}
|
||||
- HERMES_GID=${HERMES_GID:-10000}
|
||||
- GATEWAY_HEALTH_URL=http://cortex-hermes-gateway:8642
|
||||
- GATEWAY_HEALTH_URL=http://localhost:8642
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
hermes-agent gateway run &
|
||||
GATEWAY_PID=$!
|
||||
until curl -sf http://localhost:8642 >/dev/null 2>&1; do sleep 1; done
|
||||
hermes-agent dashboard --host 0.0.0.0 --no-open &
|
||||
DASHBOARD_PID=$!
|
||||
wait -n $GATEWAY_PID $DASHBOARD_PID
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8642"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
networks:
|
||||
- cortex-net
|
||||
|
||||
networks:
|
||||
cortex-net:
|
||||
driver: bridge
|
||||
Loading…
Reference in New Issue