Skip to content

Commit 55fc707

Browse files
Kalesh APsbasavapatna
Kalesh AP
authored and
sbasavapatna
committed
bnxt_re/lib: Direct Verbs: Support DBR and UMEM verbs
The following Direct Verb (DV) APIs have been implemented in this patch. Doorbell Region Direct Verbs: ----------------------------- - bnxt_re_dv_alloc_db_region(): This will allow the appliation to create extra doorbell regions and use the associated doorbell page index in dv_create_qp() and use the associated DB address while ringing the doorbell. - bnxt_re_dv_free_db_region(): Free the allocated doorbell region. - bnxt_re_dv_get_default_db_region(): Return the default doorbell page index and doorbell page address associated with the ucontext. Umem Registration Direct Verbs: ------------------------------- - bnxt_re_dv_umem_reg(): Register the user memory to be used by the application with the driver. Application can register a large chunk of memory and use it during subsequent resource creation DV APIs. Note that the driver doesn't really map/pin the user memory during dv_umem_reg(). The app specified memory params (addr, len) are saved and a corresponding umem-handle is returned. This memory is mapped/pinned when the application subsequently creates the required resources (CQ/QP) using respective direct verbs. This is implemented in the next patch in this series. - bnxt_re_dv_umem_dereg(): Deregister the user memory specified by the umem-handle. Co-developed-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> Signed-off-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com> Reviewed-by: Selvin Thyparampil Xavier <selvin.xavier@broadcom.com>
1 parent 932576e commit 55fc707

File tree

5 files changed

+367
-0
lines changed

5 files changed

+367
-0
lines changed

‎providers/bnxt_re/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ rdma_provider(bnxt_re
33
main.c
44
memory.c
55
verbs.c
6+
dv.c
67
)

‎providers/bnxt_re/bnxt_re.map‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
global:
3+
openib_driver_init;
4+
bnxt_re_dv_alloc_db_region;
5+
bnxt_re_dv_free_db_region;
6+
bnxt_re_dv_umem_reg;
7+
bnxt_re_dv_umem_dereg;
8+
bnxt_re_dv_get_default_db_region;
9+
local: *;
10+
};

‎providers/bnxt_re/bnxt_re_dv.h‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verb support user interface header
35+
*/
36+
37+
#ifndef __BNXT_RE_DV_H__
38+
#define __BNXT_RE_DV_H__
39+
40+
#include <stdint.h>
41+
#include <infiniband/verbs.h>
42+
#ifdef __cplusplus
43+
extern "C" {
44+
#endif
45+
46+
struct bnxt_re_dv_db_region_attr {
47+
uint32_t handle;
48+
uint32_t dpi;
49+
uint64_t umdbr;
50+
__u64 *dbr;
51+
};
52+
53+
enum bnxt_re_dv_umem_in_flags {
54+
BNXT_RE_DV_UMEM_FLAGS_DMABUF = 1 << 0,
55+
};
56+
57+
struct bnxt_re_dv_umem_reg_attr {
58+
void *addr;
59+
size_t size;
60+
uint32_t access_flags;
61+
uint64_t pgsz_bitmap;
62+
uint64_t comp_mask;
63+
int dmabuf_fd;
64+
};
65+
66+
struct bnxt_re_dv_db_region_attr *
67+
bnxt_re_dv_alloc_db_region(struct ibv_context *ctx);
68+
int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
69+
struct bnxt_re_dv_db_region_attr *attr);
70+
int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
71+
struct bnxt_re_dv_db_region_attr *out);
72+
void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx,
73+
struct bnxt_re_dv_umem_reg_attr *in);
74+
int bnxt_re_dv_umem_dereg(void *umem_handle);
75+
#ifdef __cplusplus
76+
}
77+
#endif
78+
#endif /* __BNXT_RE_DV_H__ */

‎providers/bnxt_re/dv.c‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verbs API function definitions.
35+
*/
36+
37+
#include <stdio.h>
38+
#include <sys/mman.h>
39+
40+
#include "main.h"
41+
#include "bnxt_re-abi.h"
42+
#include "bnxt_re_dv.h"
43+
#include "./verbs.h"
44+
#include "dv_internal.h"
45+
46+
/* Returns details about the default Doorbell page for ucontext */
47+
int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
48+
struct bnxt_re_dv_db_region_attr *out)
49+
{
50+
struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx);
51+
struct bnxt_re_dv_db_region_attr attr = {};
52+
int ret;
53+
54+
DECLARE_COMMAND_BUFFER(cmd,
55+
BNXT_RE_OBJECT_DBR,
56+
BNXT_RE_METHOD_DBR_QUERY,
57+
1);
58+
59+
fill_attr_out_ptr(cmd, BNXT_RE_DV_QUERY_DBR_ATTR, &attr);
60+
61+
ret = execute_ioctl(ibvctx, cmd);
62+
if (ret) {
63+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n", __func__, ret);
64+
return ret;
65+
}
66+
out->dbr = cntx->udpi.dbpage;
67+
out->dpi = attr.dpi;
68+
out->umdbr = attr.umdbr;
69+
return 0;
70+
}
71+
72+
int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
73+
struct bnxt_re_dv_db_region_attr *attr)
74+
{
75+
struct bnxt_re_dev *dev = to_bnxt_re_dev(ctx->device);
76+
int ret;
77+
78+
DECLARE_COMMAND_BUFFER(cmd,
79+
BNXT_RE_OBJECT_DBR,
80+
BNXT_RE_METHOD_DBR_FREE,
81+
1);
82+
83+
if (attr->dbr != MAP_FAILED)
84+
munmap(attr->dbr, dev->pg_size);
85+
86+
bnxt_trace_dv(NULL, DEV "%s: DV DBR: handle: 0x%x\n", __func__, attr->handle);
87+
fill_attr_in_obj(cmd, BNXT_RE_DV_FREE_DBR_HANDLE, attr->handle);
88+
89+
ret = execute_ioctl(ctx, cmd);
90+
if (ret) {
91+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n",
92+
__func__, ret);
93+
errno = ret;
94+
return ret;
95+
}
96+
97+
free(attr);
98+
return 0;
99+
}
100+
101+
struct bnxt_re_dv_db_region_attr *
102+
bnxt_re_dv_alloc_db_region(struct ibv_context *ctx)
103+
{
104+
struct bnxt_re_dev *dev = to_bnxt_re_dev(ctx->device);
105+
struct bnxt_re_dv_db_region_attr attr = {}, *out;
106+
struct ib_uverbs_attr *handle;
107+
uint64_t mmap_offset = 0;
108+
int ret;
109+
110+
DECLARE_COMMAND_BUFFER(cmd,
111+
BNXT_RE_OBJECT_DBR,
112+
BNXT_RE_METHOD_DBR_ALLOC,
113+
3);
114+
115+
out = calloc(1, sizeof(*out));
116+
if (!out) {
117+
errno = ENOMEM;
118+
return NULL;
119+
}
120+
121+
handle = fill_attr_out_obj(cmd, BNXT_RE_DV_ALLOC_DBR_HANDLE);
122+
fill_attr_out_ptr(cmd, BNXT_RE_DV_ALLOC_DBR_ATTR, &attr);
123+
fill_attr_out_ptr(cmd, BNXT_RE_DV_ALLOC_DBR_OFFSET, &mmap_offset);
124+
125+
ret = execute_ioctl(ctx, cmd);
126+
if (ret) {
127+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n",
128+
__func__, ret);
129+
free(out);
130+
errno = ret;
131+
return NULL;
132+
}
133+
out->handle = read_attr_obj(BNXT_RE_DV_ALLOC_DBR_HANDLE, handle);
134+
out->dpi = attr.dpi;
135+
out->umdbr = attr.umdbr;
136+
137+
out->dbr = mmap(NULL, dev->pg_size, PROT_WRITE,
138+
MAP_SHARED, ctx->cmd_fd, mmap_offset);
139+
if (out->dbr == MAP_FAILED) {
140+
fprintf(stderr, DEV "%s: mmap failed\n", __func__);
141+
bnxt_re_dv_free_db_region(ctx, out);
142+
errno = ENOMEM;
143+
return NULL;
144+
}
145+
bnxt_trace_dv(NULL, "%s: DV DBR: handle: 0x%x\n", __func__, out->handle);
146+
147+
return out;
148+
}
149+
150+
void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx, struct bnxt_re_dv_umem_reg_attr *in)
151+
{
152+
DECLARE_COMMAND_BUFFER(cmd,
153+
BNXT_RE_OBJECT_UMEM,
154+
BNXT_RE_METHOD_UMEM_REG,
155+
6);
156+
struct ib_uverbs_attr *handle;
157+
struct bnxt_re_dv_umem_internal *umem;
158+
int ret;
159+
160+
umem = calloc(1, sizeof(*umem));
161+
if (!umem) {
162+
errno = ENOMEM;
163+
return NULL;
164+
}
165+
if (ibv_dontfork_range(in->addr, in->size))
166+
goto err;
167+
168+
fill_attr_in_uint64(cmd, BNXT_RE_UMEM_OBJ_REG_ADDR, (uintptr_t)in->addr);
169+
fill_attr_in_uint64(cmd, BNXT_RE_UMEM_OBJ_REG_LEN, in->size);
170+
fill_attr_in_uint32(cmd, BNXT_RE_UMEM_OBJ_REG_ACCESS, in->access_flags);
171+
if (in->comp_mask & BNXT_RE_DV_UMEM_FLAGS_DMABUF) {
172+
if (in->dmabuf_fd == -1) {
173+
fprintf(stderr, "%s: failed: EBADF\n", __func__);
174+
errno = EBADF;
175+
goto err;
176+
}
177+
fill_attr_in_fd(cmd, BNXT_RE_UMEM_OBJ_REG_DMABUF_FD,
178+
in->dmabuf_fd);
179+
}
180+
fill_attr_in_uint64(cmd, BNXT_RE_UMEM_OBJ_REG_PGSZ_BITMAP,
181+
in->pgsz_bitmap);
182+
handle = fill_attr_out_obj(cmd, BNXT_RE_UMEM_OBJ_REG_HANDLE);
183+
184+
ret = execute_ioctl(ibvctx, cmd);
185+
if (ret) {
186+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n", __func__, ret);
187+
goto err_umem_reg_cmd;
188+
}
189+
190+
umem->handle = read_attr_obj(BNXT_RE_UMEM_OBJ_REG_HANDLE, handle);
191+
umem->context = ibvctx;
192+
umem->addr = in->addr;
193+
umem->size = in->size;
194+
195+
bnxt_trace_dv(NULL, "%s: DV Umem Reg: handle: 0x%x addr: 0x%lx size: 0x%lx\n",
196+
__func__, umem->handle, (uint64_t)umem->addr, umem->size);
197+
return (void *)umem;
198+
err_umem_reg_cmd:
199+
ibv_dofork_range(in->addr, in->size);
200+
err:
201+
free(umem);
202+
return NULL;
203+
}
204+
205+
int bnxt_re_dv_umem_dereg(void *umem_handle)
206+
{
207+
struct bnxt_re_dv_umem_internal *umem = umem_handle;
208+
209+
DECLARE_COMMAND_BUFFER(cmd,
210+
BNXT_RE_OBJECT_UMEM,
211+
BNXT_RE_METHOD_UMEM_DEREG,
212+
1);
213+
int ret;
214+
215+
bnxt_trace_dv(NULL, "%s: DV Umem Dereg: handle: 0x%x\n",
216+
__func__, umem->handle);
217+
fill_attr_in_obj(cmd, BNXT_RE_UMEM_OBJ_DEREG_HANDLE, umem->handle);
218+
ret = execute_ioctl(umem->context, cmd);
219+
if (ret) {
220+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n",
221+
__func__, ret);
222+
return ret;
223+
}
224+
225+
ibv_dofork_range(umem->addr, umem->size);
226+
free(umem);
227+
return 0;
228+
}

‎providers/bnxt_re/dv_internal.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verb support user interface header
35+
*/
36+
37+
#ifndef __BNXT_RE_DV_INTERNAL_H__
38+
#define __BNXT_RE_DV_INTERNAL_H__
39+
40+
#include <stdint.h>
41+
#include <infiniband/verbs.h>
42+
43+
struct bnxt_re_dv_umem_internal {
44+
struct ibv_context *context;
45+
uint32_t handle;
46+
void *addr;
47+
size_t size;
48+
};
49+
50+
#endif /* __BNXT_RE_DV_INTERNAL_H__ */

0 commit comments

Comments
 (0)