75 lines
2.1 KiB
YAML
75 lines
2.1 KiB
YAML
# docker-compose.yml – Hermes Agent + vLLM (Qwen3-14B-AWQ)
|
||
# Host: Cortex (2x RTX 3060 12GB)
|
||
# Daten liegen unter /opt/vllm und /opt/hermes (Sicherung/Backup)
|
||
|
||
services:
|
||
vllm:
|
||
image: vllm/vllm-openai:latest
|
||
container_name: cortex-vllm
|
||
restart: unless-stopped
|
||
runtime: nvidia
|
||
ipc: host
|
||
ports:
|
||
- "8005:8000"
|
||
environment:
|
||
- NVIDIA_VISIBLE_DEVICES=0,1
|
||
- HUGGING_FACE_HUB_TOKEN=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
|
||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||
volumes:
|
||
- /opt/vllm/huggingface:/root/.cache/huggingface
|
||
deploy:
|
||
resources:
|
||
reservations:
|
||
devices:
|
||
- driver: nvidia
|
||
count: 2
|
||
capabilities: [gpu]
|
||
command:
|
||
- --model=Qwen/Qwen3-14B-AWQ
|
||
- --quantization=awq
|
||
- --tensor-parallel-size=2
|
||
- --gpu-memory-utilization=0.82
|
||
- --max-model-len=32768
|
||
- --max-num-seqs=16
|
||
- --kv-cache-dtype=fp8
|
||
- --served-model-name=qwen3-14b-awq
|
||
- --enable-auto-tool-choice
|
||
- --tool-call-parser=hermes
|
||
- --reasoning-parser=deepseek_r1
|
||
networks:
|
||
- cortex-net
|
||
|
||
hermes-agent:
|
||
image: nousresearch/hermes-agent:latest
|
||
container_name: cortex-hermes-agent
|
||
restart: unless-stopped
|
||
depends_on:
|
||
vllm:
|
||
condition: service_healthy
|
||
shm_size: 1g
|
||
ports:
|
||
- "9119:9119" # Dashboard, jetzt im LAN erreichbar
|
||
- "8642:8642" # Gateway-API, im LAN erreichbar
|
||
volumes:
|
||
- /opt/hermes:/opt/data
|
||
environment:
|
||
- HERMES_UID=${HERMES_UID:-10000}
|
||
- HERMES_GID=${HERMES_GID:-10000}
|
||
- OPENAI_BASE_URL=http://vllm:8000/v1
|
||
- OPENAI_API_KEY=dummy
|
||
- GATEWAY_HEALTH_URL=http://localhost:8642
|
||
entrypoint: ["/bin/sh", "-c"]
|
||
command:
|
||
- |
|
||
hermes-agent gateway run &
|
||
GATEWAY_PID=$!
|
||
until curl -sf http://localhost:8642 >/dev/null 2>&1; do sleep 1; done
|
||
hermes-agent dashboard --host 0.0.0.0 --no-open &
|
||
DASHBOARD_PID=$!
|
||
wait -n $GATEWAY_PID $DASHBOARD_PID
|
||
networks:
|
||
- cortex-net
|
||
|
||
networks:
|
||
cortex-net:
|
||
driver: bridge |