services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    restart: unless-stopped
    ipc: host
    ports:
      - "8000:8000"
    volumes:
      - /opt/vLLM/models:/root/.cache/huggingface
      - /opt/vLLM/config:/vllm-workspace/config
    environment:
      - HUGGING_FACE_HUB_TOKEN=hf_12345
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    command: >
      --model Qwen/Qwen2.5-7B-Instruct-AWQ
      --host 0.0.0.0
      --port 8000
      --gpu-memory-utilization 0.85
      --max-model-len 65536
      --hf-overrides {"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
      --enable-auto-tool-choice
      --tool-call-parser hermes