composte-hermes.yml aktualisiert

composte-hermes.yml

@@ -11,7 +11,6 @@ services:
       - NVIDIA_VISIBLE_DEVICES=0,1
       - HUGGING_FACE_HUB_TOKEN=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
       - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
     volumes:
       - /opt/vllm/huggingface:/root/.cache/huggingface
     deploy:
@@ -23,16 +22,18 @@ services:
               capabilities: [gpu]
     command:
       - --model=Qwen/Qwen3-14B-AWQ
+      - --port=8005
+      - --quantization=awq
       - --tensor-parallel-size=2
-       - --gpu-memory-utilization=0.92
+      - --gpu-memory-utilization=0.82
       - --max-model-len=65536
-       - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}}
+      - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
-       - --enable-prefix-caching
+      - --max-num-seqs=16
+      - --kv-cache-dtype=fp8
+      - --served-model-name=qwen3-14b-awq
       - --enable-auto-tool-choice
       - --tool-call-parser=hermes
-       - --max-num-seqs=4
+      - --reasoning-parser=deepseek_r1
-       - --generation-config=vllm
-       - --disable-custom-all-reduce
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
       interval: 30s
@@ -57,6 +58,7 @@ services:
     volumes:
       - /opt/hermes:/opt/data
       - /opt/hermes/.ssh:/root/.ssh
+      - /var/run/docker.sock:/var/run/docker.sock
 
     environment:
       - OPENAI_BASE_URL=http://vllm:8005/v1