88 lines
2.3 KiB
YAML
88 lines
2.3 KiB
YAML
services:
|
|
vllm:
|
|
image: vllm/vllm-openai:latest
|
|
container_name: cortex-vllm
|
|
restart: unless-stopped
|
|
runtime: nvidia
|
|
ipc: host
|
|
ports:
|
|
- "8005:8005"
|
|
environment:
|
|
- NVIDIA_VISIBLE_DEVICES=0,1
|
|
- HUGGING_FACE_HUB_TOKEN=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
|
|
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
|
volumes:
|
|
- /opt/vllm/huggingface:/root/.cache/huggingface
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: 2
|
|
capabilities: [gpu]
|
|
command:
|
|
- --model=Qwen/Qwen3-14B-AWQ
|
|
- --port=8005
|
|
- --quantization=awq
|
|
- --tensor-parallel-size=2
|
|
- --gpu-memory-utilization=0.84
|
|
- --max-model-len=65536
|
|
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
|
|
- --max-num-seqs=4
|
|
- --served-model-name=qwen3-14b-awq
|
|
- --enable-auto-tool-choice
|
|
- --tool-call-parser=hermes
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 180s
|
|
networks:
|
|
- cortex-net
|
|
|
|
hermes-gateway:
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
image: nousresearch/hermes-agent:latest
|
|
container_name: cortex-hermes-gateway
|
|
restart: unless-stopped
|
|
user: "0:0"
|
|
privileged: true
|
|
depends_on:
|
|
vllm:
|
|
condition: service_healthy
|
|
command: ["gateway", "run"]
|
|
volumes:
|
|
- /opt/hermes:/opt/data
|
|
- /opt/hermes/.ssh:/root/.ssh
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
|
|
environment:
|
|
- OPENAI_BASE_URL=http://vllm:8005/v1
|
|
- OPENAI_API_KEY=sk-dummy-key-1234567890
|
|
networks:
|
|
- cortex-net
|
|
|
|
hermes-dashboard:
|
|
user: "0:0"
|
|
image: nousresearch/hermes-agent:latest
|
|
container_name: cortex-hermes-dashboard
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- hermes-gateway
|
|
command: ["dashboard", "--host", "0.0.0.0", "--no-open"]
|
|
ports:
|
|
- "9119:9119"
|
|
volumes:
|
|
- /opt/hermes:/opt/data
|
|
- /opt/hermes/.ssh:/root/.ssh
|
|
|
|
environment:
|
|
- GATEWAY_HEALTH_URL=http://cortex-hermes-gateway:8642
|
|
networks:
|
|
- cortex-net
|
|
|
|
networks:
|
|
cortex-net:
|
|
driver: bridge |