diff --git a/docker-compose.yml b/docker-compose.yml index 794c0d2..fc801c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,14 @@ services: network_mode: bridge environment: - HF_TOKEN=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 -# - PYTORCH_ALLOC_CONF=expandable_segments:True -# - VLLM_DEVICE=cpu + - VLLM_GPU_MEMORY_UTILIZATION=0.97 + - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 ipc: host - command: --model Qwen/Qwen2.5-14B-Instruct-AWQ + command: > + vllm serve + Qwen/Qwen2.5-14B-Instruct-AWQ + --max-model-len 8192 + --gpu-memory-utilization 0.97 ports: - "8000:8000" volumes: