compose-vLLM.yaml aktualisiert

compose-vLLM.yaml

@@ -3,20 +3,17 @@ services:
     image: vllm/vllm-openai:latest
     container_name: vllm
     restart: unless-stopped
-    ipc: host # vLLM braucht shared memory für Tensor-Parallelism
-
+    ipc: host
     ports:
-      - "8000:8000" # OpenAI-kompatible API
-
+      - "8000:8000"
     volumes:
-      - /opt/vLLM/models:/root/.cache/huggingface # HF-Model-Cache
+      - /opt/vLLM/models:/root/.cache/huggingface
       - /opt/vLLM/config:/vllm-workspace/config
-
     environment:
-      - HUGGING_FACE_HUB_TOKEN=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
+      - HUGGING_FACE_HUB_TOKEN=hf_12345
       - NVIDIA_VISIBLE_DEVICES=all
       - NVIDIA_DRIVER_CAPABILITIES=compute,utility
-
+      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
     deploy:
       resources:
         reservations:
@@ -24,12 +21,12 @@ services:
             - driver: nvidia
               count: all
               capabilities: [gpu]
-
     command: >
       --model Qwen/Qwen2.5-7B-Instruct-AWQ
       --host 0.0.0.0
       --port 8000
       --gpu-memory-utilization 0.85
       --max-model-len 65536
+      --hf-overrides {"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
       --enable-auto-tool-choice
       --tool-call-parser hermes