89 lines
2.5 KiB
YAML
89 lines
2.5 KiB
YAML
services:
|
|
vllm:
|
|
image: vllm/vllm-openai:latest
|
|
container_name: cortex-vllm
|
|
restart: unless-stopped
|
|
runtime: nvidia
|
|
ipc: host
|
|
ports:
|
|
- "8005:8005"
|
|
environment:
|
|
- NVIDIA_VISIBLE_DEVICES=0,1
|
|
- HUGGING_FACE_HUB_TOKEN=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
|
|
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
|
volumes:
|
|
- /opt/vllm/huggingface:/root/.cache/huggingface
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: 2
|
|
capabilities: [gpu]
|
|
command:
|
|
- --model=Qwen/Qwen3-14B-AWQ
|
|
- --port=8005
|
|
- --quantization=awq
|
|
- --tensor-parallel-size=2
|
|
- --gpu-memory-utilization=0.82
|
|
- --max-model-len=65536
|
|
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
|
|
- --max-num-seqs=16
|
|
- --kv-cache-dtype=fp8
|
|
- --served-model-name=qwen3-14b-awq
|
|
- --enable-auto-tool-choice
|
|
- --tool-call-parser=hermes
|
|
- --reasoning-parser=deepseek_r1
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 180s
|
|
networks:
|
|
- cortex-net
|
|
|
|
hermes-agent:
|
|
image: nousresearch/hermes-agent:latest
|
|
container_name: cortex-hermes-agent
|
|
restart: unless-stopped
|
|
depends_on:
|
|
vllm:
|
|
condition: service_healthy
|
|
shm_size: 1g
|
|
ports:
|
|
- "9119:9119"
|
|
- "8642:8642"
|
|
volumes:
|
|
- /opt/hermes:/opt/data
|
|
environment:
|
|
- HERMES_UID=${HERMES_UID:-10000}
|
|
- HERMES_GID=${HERMES_GID:-10000}
|
|
- OPENAI_BASE_URL=http://vllm:8005/v1
|
|
- OPENAI_API_KEY=sk-dummy-key-1234567890
|
|
- GATEWAY_HEALTH_URL=http://localhost:8642
|
|
entrypoint: ["/bin/bash", "-c"]
|
|
command:
|
|
- |
|
|
set -e
|
|
hermes-agent gateway run &
|
|
GATEWAY_PID=$!
|
|
until curl -sf http://localhost:8642 >/dev/null 2>&1; do sleep 1; done
|
|
hermes-agent dashboard --host 0.0.0.0 --no-open &
|
|
DASHBOARD_PID=$!
|
|
wait -n $GATEWAY_PID $DASHBOARD_PID
|
|
EXIT_CODE=$?
|
|
kill $GATEWAY_PID $DASHBOARD_PID 2>/dev/null || true
|
|
exit $EXIT_CODE
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8642 && curl -f http://localhost:9119"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 60s
|
|
networks:
|
|
- cortex-net
|
|
|
|
networks:
|
|
cortex-net:
|
|
driver: bridge |