1，搭建环境

1.1 一键搭建 ucx

git clone https://github.com/openucx/ucx.git && \
cd ucx/ &&\
​git checkout v1.17.0 && \
./autogen.sh && \
./autogen.sh && \
mkdir build && \
cd build && \
../contrib/configure-devel --prefix=${PWD}/../ --enable-debug --enable-gtest --with-cuda=/usr/local/cuda --without-rocm --without-knem --without-java && \
make -j && \
make install

1.2 一键搭建 hwloc

git checkout hwloc-2.11.2 &&\
./autogen.sh &&\
mkdir build &&\
cd build &&\
../configure --prefix=/home/hanmeimei/ex_hwloc/tmp/localhw --enable-debug  --with-cuda=/usr/local/cuda  --with-cuda-version=12.4  &&\
make -j &&\
make install

1.3 一键搭建 openmpi

git clone https://github.com/open-mpi/ompi.git &&\
cd ompi &&\
git checkout v5.0.6 &&\
git submodule update --recursive --init &&\
./autogen.pl &&\
mkdir build &&\
cd build &&\
../configure --prefix=${PWD}/../../local_ompi_hwloc --enable-debug --with-tests-examples  --with-hwloc=/home/hanmeimei/ex_hwloc/tmp/localhw  --with-hwloc-libdir=/home/hongleili/ex_hwloc/tmp/localhw/lib --with-ucx=/home/hanmeimei/ex_openmpi/tmp5_runtest/tmp1/ucx --without-verbs &&\
make -j &&\
make install

2. 跑一个gpu相关的 hwloc case

/* This example program plays with:
 * - finding GPU OS devices
 * - getting CUDA and OpenCL attributes
 * - displaying the locality of the GPU
 *
 * Copyright © 2009-2019 Inria.  All rights reserved.
 * Copyright © 2009-2011,2017 Université Bordeaux
 * Copyright © 2009-2010 Cisco Systems, Inc.  All rights reserved.
 * See COPYING in top-level directory.
 */

#include "hwloc.h"

#include <errno.h>
#include <stdio.h>
#include <string.h>

int main(void)
{
    hwloc_topology_t topology;
    hwloc_obj_t obj;
    unsigned n, i;
    int devid, platformid;
    const char *dev;

    /* Allocate, initialize and load topology object. */
    hwloc_topology_init(&topology);
    hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
    hwloc_topology_load(topology);

    /* Find CUDA devices through the corresponding OS devices */
    n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);

    for (i = 0; i < n ; i++) {
      const char *s;
      obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_OS_DEVICE, i);
      printf("%s:\n", obj->name);

      /* obj->attr->osdev.type is HWLOC_OBJ_OSDEV_COPROC */

      s = hwloc_obj_get_info_by_name(obj, "Backend");
      /* obj->subtype also contains CUDA or OpenCL since v2.0 */

      if (s && !strcmp(s, "CUDA")) {
        /* This is a CUDA device */
        assert(!strncmp(obj->name, "cuda", 4));
        devid = atoi(obj->name + 4);
        printf("CUDA device %d\n", devid);

        s = hwloc_obj_get_info_by_name(obj, "GPUModel");
        if (s)
          printf("Model: %s\n", s);

        s = hwloc_obj_get_info_by_name(obj, "CUDAGlobalMemorySize");
        if (s)
          printf("Memory: %s\n", s);

        s = hwloc_obj_get_info_by_name(obj, "CUDAMultiProcessors");
        if (s)
        {
          int mp = atoi(s);
          s = hwloc_obj_get_info_by_name(obj, "CUDACoresPerMP");
          if (s) {
            int mp_cores = atoi(s);
            printf("Cores: %d\n", mp * mp_cores);
          }
        }
      }

      if (s && !strcmp(s, "OpenCL")) {
        /* This is an OpenCL device */
        assert(!strncmp(obj->name, "opencl", 6));
        platformid = atoi(obj->name + 6);
        printf("OpenCL platform %d\n", platformid);
        dev = strchr(obj->name + 6, 'd');
        devid = atoi(dev + 1);
        printf("OpenCL device %d\n", devid);

        s = hwloc_obj_get_info_by_name(obj, "GPUModel");
        if (s)
          printf("Model: %s\n", s);

        s = hwloc_obj_get_info_by_name(obj, "OpenCLGlobalMemorySize");
        if (s)
          printf("Memory: %s\n", s);
      }

      /* One can also use helpers from hwloc/cuda.h, hwloc/cudart.h,
       * hwloc/opencl.h */


      /* Find out cpuset this is connected to */
      while (obj && (!obj->cpuset || hwloc_bitmap_iszero(obj->cpuset)))
        obj = obj->parent;

      if (obj) {
        char *cpuset_string;
        char name[16];
        hwloc_obj_type_snprintf(name, sizeof(name), obj, 0);
        hwloc_bitmap_asprintf(&cpuset_string, obj->cpuset);
        printf("Location: %s P#%u\n", name, obj->os_index);
        printf("Cpuset: %s\n", cpuset_string);
      }
      printf("\n");
    }

    /* Destroy topology object. */
    hwloc_topology_destroy(topology);

    return 0;
}

Makefile:

CC := /home/hanmeimei/ex_hwloc/tmp/local_ompi_hwloc/bin/mpicc

all: hello

%: %.c
	$(CC) -g $< -o $@ $(INC) $(LD_FLAGS)
 
INC := -I /usr/local/cuda/include
LD_FLAGS := -lhwloc -L/usr/local/cuda/lib64 -lcuda


.PHONY: clean
clean:
	-rm -rf hello

$ make

$ ./hello

全部输出如下：

2路cpu，8A100卡

$ ./hello
enp97s0f0:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff

mlx5_4:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff

enp97s0f1:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff

mlx5_5:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff

opencl0d2:
OpenCL platform 0
OpenCL device 2
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

ibp75s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

mlx5_2:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

opencl0d3:
OpenCL platform 0
OpenCL device 3
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

nvme4c4n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

nvme5c5n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

ibp84s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

mlx5_3:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000

nvme2n1:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,0x0

nvme3n1:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,0x0

opencl0d0:
OpenCL platform 0
OpenCL device 0
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

nvme0c0n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

nvme1c1n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

ibp12s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

mlx5_0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

opencl0d1:
OpenCL platform 0
OpenCL device 1
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

ibp18s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

mlx5_1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,0x0

enp225s0f0:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,,0x0

mlx5_10:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,,0x0

enp225s0f1:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,,0x0

mlx5_11:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,,0x0

enp226s0:
Location: Group0 P#4294967295
Cpuset: 0x0000ffff,,,,0x0000ffff,,0x0

opencl0d6:
OpenCL platform 0
OpenCL device 6
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

ibp186s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

mlx5_8:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

opencl0d7:
OpenCL platform 0
OpenCL device 7
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

nvme8c8n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

nvme9c9n1:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

ibp204s0:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

mlx5_9:
Location: Group0 P#4294967295
Cpuset: 0xffff0000,,,,0xffff0000,,0x0

opencl0d4:
OpenCL platform 0
OpenCL device 4
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

nvme6c6n1:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

nvme7c7n1:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

ibp141s0:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

mlx5_6:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

opencl0d5:
OpenCL platform 0
OpenCL device 5
Model: NVIDIA A100-SXM4-80GB
Memory: 83103488
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

ibp148s0:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

mlx5_7:
Location: Group0 P#4294967295
Cpuset: 0x7fff0000,,,,0xffff0000,,,0x0

3，选择gpu的 源代码

（openmpi官方指导版本，有问题，待解）

hello_sel_node_gpu.c

/**
 * Test program to show the use of hwloc to select the GPU closest to the CPU
 * that the MPI program is running on.  Note that this works even without
 * any libpciaccess or libpci support as it keys off the NVIDIA vendor ID.
 * There may be other ways to implement this but this is one way.
 * January 10, 2014
 */
#include <assert.h>
#include <stdio.h>
#include "cuda.h"
#include "mpi.h"
#include "hwloc.h"

#define ABORT_ON_ERROR(func) \
  { CUresult res; \
    res = func; \
    if (CUDA_SUCCESS != res) { \
        printf("%s returned error=%d\n", #func, res); \
        abort(); \
    } \
  }
static hwloc_topology_t topology = NULL;
static int gpuIndex = 0;
static hwloc_obj_t gpus[16] = {0};

/**
 * This function searches for all the GPUs that are hanging off a NUMA
 * node.  It walks through each of the PCI devices and looks for ones
 * with the NVIDIA vendor ID.  It then stores them into an array.
 * Note that there can be more than one GPU on the NUMA node.
 */
static void find_gpus(hwloc_topology_t topology, hwloc_obj_t parent, hwloc_obj_t child) {
    hwloc_obj_t pcidev;
    pcidev = hwloc_get_next_child(topology, parent, child);
    if (NULL == pcidev) {
        return;
    } else if (0 != pcidev->arity) {
        /* This device has children so need to look recursively at them */
        find_gpus(topology, pcidev, NULL);
        find_gpus(topology, parent, pcidev);
    } else {
        if (pcidev->attr->pcidev.vendor_id == 0x10de) {
            gpus[gpuIndex++] = pcidev;
        }
        find_gpus(topology, parent, pcidev);
    }
}

int main(int argc, char *argv[])
{
    int rank, retval, length;
    char procname[MPI_MAX_PROCESSOR_NAME+1];
    const unsigned long flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES;
    hwloc_cpuset_t newset;
    hwloc_obj_t node, bridge;
    char pciBusId[16];
    CUdevice dev;
    char devName[256];

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    if (MPI_SUCCESS != MPI_Get_processor_name(procname, &length)) {
        strcpy(procname, "unknown");
    }

    /* Now decide which GPU to pick.  This requires hwloc to work properly.
     * We first see which CPU we are bound to, then try and find a GPU nearby.
     */
    retval = hwloc_topology_init(&topology);
    assert(retval == 0);
    retval = hwloc_topology_set_flags(topology, flags);
    assert(retval == 0);
    retval = hwloc_topology_load(topology);
    assert(retval == 0);
    newset = hwloc_bitmap_alloc();
    retval = hwloc_get_last_cpu_location(topology, newset, 0);
    assert(retval == 0);

    /* Get the object that contains the cpuset */
    node = hwloc_get_first_largest_obj_inside_cpuset(topology, newset);

    /* Climb up from that object until we find the HWLOC_OBJ_NODE */
    while (node->type != HWLOC_OBJ_NODE) {
        node = node->parent;
    }

    /* Now look for the HWLOC_OBJ_BRIDGE.  All PCI busses hanging off the
     * node will have one of these */
    bridge = hwloc_get_next_child(topology, node, NULL);
    while (bridge->type != HWLOC_OBJ_BRIDGE) {
        bridge = hwloc_get_next_child(topology, node, bridge);
    }

    /* Now find all the GPUs on this NUMA node and put them into an array */
    find_gpus(topology, bridge, NULL);

    ABORT_ON_ERROR(cuInit(0));
    /* Now select the first GPU that we find */
    if (gpus[0] == 0) {
        printf("No GPU found\n");
    } else {
        sprintf(pciBusId, "%.2x:%.2x:%.2x.%x", gpus[0]->attr->pcidev.domain, gpus[0]->attr->pcidev.bus,
        gpus[0]->attr->pcidev.dev, gpus[0]->attr->pcidev.func);
        ABORT_ON_ERROR(cuDeviceGetByPCIBusId(&dev, pciBusId));
        ABORT_ON_ERROR(cuDeviceGetName(devName, 256, dev));
        printf("rank=%d (%s): Selected GPU=%s, name=%s\n", rank, procname, pciBusId, devName);
    }

    MPI_Finalize();
    return 0;
}