diff --git a/compose-vLLM.yaml b/compose-vLLM.yaml index b6e16de..8eb51ed 100644 --- a/compose-vLLM.yaml +++ b/compose-vLLM.yaml @@ -26,10 +26,10 @@ services: capabilities: [gpu] command: > - --model Qwen/Qwen2.5-14B-Instruct-AWQ + --model Qwen/Qwen2.5-7B-Instruct-AWQ --host 0.0.0.0 --port 8000 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.75 --max-model-len 16384 --enable-auto-tool-choice --tool-call-parser hermes \ No newline at end of file