services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    restart: unless-stopped
    ipc: host
    ports:
      - "8000:8000"
    volumes:
      - /opt/vLLM/models:/root/.cache/huggingface
      - /opt/vLLM/config:/vllm-workspace/config
    environment:
      - HUGGING_FACE_HUB_TOKEN=hf_12345
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    command:
      - "--model"
      - "Qwen/Qwen2.5-7B-Instruct-AWQ"
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "8000"
      - "--gpu-memory-utilization"
      - "0.85"
      - "--max-model-len"
      - "65536"
      - "--hf-overrides"
      - '{"rope_scaling": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 32768}}'
      - "--enable-auto-tool-choice"
      - "--tool-call-parser"
      - "hermes"