Initial

compose-vLLM.yml

@@ -0,0 +1,46 @@
+services:
+  vllm:
+    image: vllm/vllm-openai:latest
+    container_name: vllm
+    restart: unless-stopped
+
+    ipc: host
+    gpus: all
+
+    ports:
+      - "8000:8000"
+
+    volumes:
+      - /opt/vLLM/models:/root/.cache/huggingface
+      - /opt/vLLM/config:/vllm-workspace/config
+
+    environment:
+      HUGGING_FACE_HUB_TOKEN: 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
+      NVIDIA_VISIBLE_DEVICES: "0,1"
+      NVIDIA_DRIVER_CAPABILITIES: compute,utility
+
+    shm_size: "16g"
+
+    command:
+      - "--model"
+      - "Qwen/Qwen3-14B-AWQ"
+      - "--served-model-name"
+      - "qwen3-14b"
+      - "--host"
+      - "0.0.0.0"
+      - "--port"
+      - "8000"
+      - "--tensor-parallel-size"
+      - "2"
+      - "--dtype"
+      - "half"
+      - "--gpu-memory-utilization"
+      - "0.95"
+      - "--max-model-len"
+      - "16384"
+      - "--max-num-seqs"
+      - "32"
+      - "--enable-prefix-caching"
+      - "--enable-auto-tool-choice"
+      - "--tool-call-parser"
+      - "hermes"