diff --git a/compose-vLLM.yml b/compose-vLLM.yml index afe8c17..9b99493 100644 --- a/compose-vLLM.yml +++ b/compose-vLLM.yml @@ -38,8 +38,8 @@ services: - "0.90" - "--max-model-len" - "65536" - - "--rope-scaling" - - '{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}' + - "--hf-overrides" + - '{"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}}' - "--max-num-seqs" - "16" - "--kv-cache-dtype"