问题由来

DPDK 单worker线程使用定时器处理会话超时，会有多影响性能 ？
那么今天就来研究一下 rte_timer的性能到底如何。

测试方法

普通会话超时处理逻辑：

普通会话一直在活跃，那么需要每1分钟去check以下是否活跃，如果活跃，那么就再次设定1分钟超时；如果检查到不活跃时间不足1分钟，那么再次设定剩余时间来超时；如果不活跃时间超过1分钟（idle timeout），则删除会话。在大量用户一直活跃的时候，定时器压力最大。

我们模拟100万会话，一直循环超时回调。

set 100万个定时器，大约62s内均匀插入。

超时时间以后，触发callback, reset这个定时器， 所有定时器处在不断回调不断reset的循环之中。

每隔 1ms/ 10ms 轮询一次定时器

初始化set 100万个定时器以后，再用perf record进行记录。

测试代码

#include <stdio.h>
#include <stdint.h>
#include <pthread.h>
#include <signal.h>
#include <unistd.h>
#include <sys/time.h>

#include <rte_mempool.h>
#include <rte_timer.h>
#include <rte_eal.h>
#include <rte_launch.h>
#include <rte_cycles.h>
#include <rte_malloc.h>
#include <rte_lcore.h>

#define NUM_WORKERS 5
#define NUM_TIMERS_PER_WORKER 1000000
#define TIMER_RATE_PER_SECOND 25000


#define TIMEOUT_SEC 60  // 超时秒


typedef struct {
    struct rte_timer timer; // DPDK 定时器
    uint64_t id;            
} timer_data;

int quit = 0;

#define TIMER_NUM (NUM_WORKERS * NUM_TIMERS_PER_WORKER)

struct rte_mempool *timer_mempool;

static void signal_handler(int signal) {
    switch (signal) {
    case SIGINT:
    case SIGTERM:
        printf("\n\nSignal %d received, preparing to exit...\n", signal);
        quit = true;
        break;

    default:
        break;
    }
}

// 定时器回调函数
static void timer_callback(struct rte_timer *timer, void *arg) {
    timer_data *data = (timer_data *)arg;

    // 重设定时器
    //printf("timer_callback\n");
    rte_timer_reset(timer, TIMEOUT_SEC * rte_get_timer_hz(), SINGLE, rte_lcore_id(), timer_callback, data);
}

static inline int set_timer(int lcore_id)
{   
    int ret = 0;
    void* td;
    ret = rte_mempool_get(timer_mempool, &td);
    if (ret != 0) {
        printf("Error: get timer from pool failed\n");
        return -1;
    }
    ret = rte_timer_reset(&((timer_data*)td)->timer, TIMEOUT_SEC* rte_get_timer_hz(), SINGLE, lcore_id, timer_callback, td);
    if (ret != 0) {
        printf("Error: rte_timer_reset failed %d\n", lcore_id);
        return -1;
    }
    return 0;
}

static int set_timers_worker(__rte_unused void *dummy)
{
    printf("Starting set_timer_worker on core %u\n", rte_lcore_id());
    struct timeval start_time, end_time;
    gettimeofday(&start_time, NULL);
    uint64_t start_tick = rte_rdtsc();
    
    int i = 0, ret = 0; 
    int lcore_id = rte_lcore_id();
    for (i = 0; i < NUM_TIMERS_PER_WORKER; i++) {
        ret = set_timer(lcore_id);
        if (ret != 0) {
            return -1;
        }
        usleep(10);
    }
    uint64_t end_tick = rte_rdtsc();

    gettimeofday(&end_time, NULL);
    double elapsed_time = (end_time.tv_sec - start_time.tv_sec) +
            (end_time.tv_usec - start_time.tv_usec) / 1000000.0;
    uint64_t elapsed_tick = end_tick - start_tick;
    double elapsed_s = (elapsed_tick*1000 / rte_get_timer_hz())/1000.0;

    printf("End timer_worker on core %u, cost %.3f  %.3f\n", rte_lcore_id(), elapsed_time, elapsed_s);
    while (!quit) {
        rte_timer_manage();
        rte_delay_ms(10);
    }
    return 0;
}
/*
*    command:
*    ./timertest -l 1 -n 2 --proc-type=auto
*/
int main(int argc, char *argv[]) {
    int ret;
    pthread_t worker_threads[NUM_WORKERS];
    uint64_t core_id;

    // 初始化 DPDK 环境
    ret = rte_eal_init(argc, argv);
    if (ret < 0) {
        rte_exit(EXIT_FAILURE, "EAL initialization failed\n");
    }

    // 初始化定时器运行环境
    rte_timer_subsystem_init();


    // 创建一个内存池来存储定时器数据
    timer_mempool = rte_mempool_create("timer_mempool", TIMER_NUM,  sizeof(timer_data), 0, 0, NULL, NULL, 0, 0, rte_socket_id(), 0);
    if (timer_mempool == NULL) {
        rte_exit(EXIT_FAILURE, "Failed to create mempool\n");
    }

    set_timers_worker(NULL);

    return 0;
}

Makefile

APP_tool = timertest

#SRCS-y = $(wildcard *.c)
SRCS-t = $(wildcard *.c) 
PKGCONF ?= pkg-config

CFLAGS += -g -O2 $(shell $(PKGCONF) --cflags libdpdk) 
LDFLAGS += $(shell $(PKGCONF) --libs libdpdk) -lpthread

all: $(APP_tool)

$(APP_tool): $(SRCS-t)
	$(CC) $(CFLAGS) $(SRCS-t) -o $@ $(LDFLAGS)

.PHONY: clean
clean:
	rm $(APP_tool)

测试步骤

先启动进程，等待100万定时器插入完成

root@r750-132:/home/ckun/ws/rte_timer# ./timertest -l 1 -n 2 --proc-type=auto 
EAL: Detected 32 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Auto-detected process type: PRIMARY
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.0 (socket 1)
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.1 (socket 1)
EAL: No legacy callbacks, legacy socket not created
Starting set_timer_worker on core 1
End timer_worker on core 1, cost 62.499  62.352

使用perf工具，记录运行状况，最后用工具生成火焰图

火焰图工具地址：
https://github.com/brendangregg/FlameGraph

root@r750-132:~/perfs# perf record -e cpu-clock -g -p 1892465
^C[ perf record: Woken up 106 times to write data ]
[ perf record: Captured and wrote 26.440 MB perf.data (432308 samples) ]

root@r750-132:~/perfs# ll
total 27260
drwxr-xr-x  2 root root     4096 Jan  7 17:08 ./
drwx------ 17 root root     4096 Jan  7 16:39 ../
-rwxr-xr-x  1 root root    39741 Jan  7 16:36 flamegraph.pl*
-rw-r--r--  1 root root    18960 Jan  7 17:05 perf-10-60-1.svg
-rw-r--r--  1 root root    18765 Jan  7 16:56 perf-10.svg
-rw-r--r--  1 root root    18767 Jan  7 16:43 perf-1.svg
-rw-r--r--  1 root root    18963 Jan  7 16:51 perf-2.svg
-rw-------  1 root root 27766360 Jan  7 17:10 perf.data
-rwxr-xr-x  1 root root    13234 Jan  7 16:35 stackcollapse-perf.pl*
root@r750-132:~/perfs# perf script -i perf.data &> perf.unfold
root@r750-132:~/perfs# ./stackcollapse-perf.pl perf.unfold &> perf.folded
root@r750-132:~/perfs# ./flamegraph.pl perf.folded > perf-10-60-60.svg

测试结果记录

超时5秒， 回调后reset 1秒， 轮询间隔 1 ms

rte-timer_reset直接超过 10%

超时5秒， 回调后reset 1秒， 轮询间隔 2 ms

rte-timer_reset超过 11%

超时5秒， 回调后reset 1秒， 轮询间隔 10 ms

rte-timer_reset超过 11%

超时60秒，回调后reset 1秒，轮询间隔 10ms

rte-timer_reset 下降到9%以下

超时60秒，回调后reset 60秒，轮询间隔 10ms

timer_reset 0.23%， timer相关总耗时不到0.4%， 可以忽略不计。

结论

1分钟作为循环超时时间，100万定时器的轮询回调损耗小于0.4%。在处理会话idle_timeout时，这种损耗可以忽略不计了。