pytorch 对应下载地址

Previous PyTorch Versions | PyTorch

qwen文档

https://qwen.readthedocs.io/zh-cn/latest/deployment/vllm.html

安装vllm

pip install 'vllm==0.6.1' --extra-index-url https://download.pytorch.org/whl/${CUDA_VERSION}

启动命令

python -m vllm.entrypoints.openai.api_server --served-model-name Qwen2.5-7B-Instruct --model /home/sky/model_data/Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 8080

curl 测试1

curl http://localhost:8080/v1/models

curl 测试2

curl http://localhost:8080/v1/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "Qwen2.5-7B-Instruct",
        "prompt": "请介绍一下北京，然后推荐一下当地的美食",
        "max_tokens": 512,
        "temperature": 0
    }'

python脚本测试

import requests
import time


headers = {
    'Content-Type': 'application/json',
}

json_data = {
    'model': 'Qwen2.5-7B-Instruct',
    'prompt': '请介绍一下北京，然后推荐一下当地的美食',
    'max_tokens': 100,
    'temperature': 0,
}
start_time = time.time()
response = requests.post('http://192.168.132.215:8080/v1/completions', headers=headers, json=json_data)
print(response.text)
end_time = time.time()
print(end_time - start_time)

python脚本并发测试

import aiohttp
import asyncio
import time


async def fetch(session, url, json_data):
    async with session.post(url, json=json_data) as response:
        return await response.text()


async def send_request(session,url,prompt):
    json_data = {
        'model': 'Qwen2.5-7B-Instruct',
        'prompt': prompt,
        'max_tokens': 100,
        'temperature': 0,
    }
    return await fetch(session, url, json_data)


async def main():
    headers = {
        'Content-Type': 'application/json',
    }

    # 假设我们有5个不同的提示
    prompts = [
        '请介绍一下北京，然后推荐一下当地的美食',
        '请介绍上海的文化特色和著名景点',
        '描述一下广州的气候和最佳旅游时间',
        '讲述成都的历史背景和当地风俗',
        '谈谈深圳的发展历程和经济特点',
        '请介绍一下河南',
        '请介绍一下南京',
        '请介绍一下杭州',
        '请介绍一下日本',
        '请介绍一下泰国'
    ]

    url = 'http://192.168.132.215:8080/v1/completions'
    start_time = time.time()

    async with aiohttp.ClientSession(headers=headers) as session:
        # 使用 asyncio.gather 并发运行所有 send_request 调用
        responses = await asyncio.gather(*[send_request(session,url,prompt) for prompt in prompts])

        for response in responses:
            print("**********************************************8")
            print(response)

    end_time = time.time()
    print(end_time - start_time)


# 运行事件循环
if __name__ == '__main__':
    asyncio.run(main())