1.下载

git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

2.根据cpu和cuda编译

🖥️ CPU 版本

cmake -B build_cpu
cmake --build build_cpu --config Release

🖥️ CUDA 版本

cmake -B build_cuda -DLLAMA_CUDA=ON
cmake --build build_cuda --config Release -j 12

  cmake -B build
  cmake --build build --config Release -t llama-server

3.量化

1.将 safetensors 格式转成 gguf

cd ~/code/llama.cpp/build_cuda/bin

python convert_hf_to_gguf.py /mnt/workspace/models/Qwen2.5-1.5B-Instruct --outfile /mnt/workspace/Qwen2.5-1.5B-Instruct-q8_0-v1.gguf --outtype q8_0

 2.将 gguf 格式进行（再）量化

cd ~/code/llama.cpp/build_cuda/bin

./quantize --allow-requantize /root/autodl-tmp/models/Llama3-8B-Chinese-Chat-GGUF/Llama3-8B-Chinese-Chat-q8_0-v2_1.gguf /root/autodl-tmp/models/Llama3-8B-Chinese-Chat-GGUF/Llama3-8B-Chinese-Chat-q4_1-v1.gguf Q4_1

4.部署服务：

cd llama.cpp/build/bin

./llama-server -m /mnt/workspace/Qwen2.5-7B-Instruct-GGUF/Qwen2.5-7B-Instruct-q8_0-v1.gguf/Qwen2.5-7B-Instruct-Q8_0.gguf --port 8080

5.对话：

cd llama.cpp/build/bin

./llama-cli -m /mnt/workspace/models/Qwen2.5-1.5B-Instruct-q8_0-v1.gguf     -co -cnv -p "You are a helpful assistant."     -fa -ngl 80 -n 512
​

互动：

./llama-cli -m qwen2-7b-instruct-q5_k_m.gguf \
    -co -sp -i -if -p "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" \
    --in-prefix "<|im_start|>user\n" --in-suffix "<|im_end|>\n<|im_start|>assistant\n" \
    -fa -ngl 80 -n 512