Skip to content

Commit 4c846bc

Browse files
dcherednikdcherednik
committed
[ICRDMA] hide "verbs.h" in the implementation. EXT-1571 (#27894)
Verbs headers are platform specific so we do not want to export its. Otherwise it can cause build issues.
1 parent 473bba4 commit 4c846bc

File tree

9 files changed

+122
-46
lines changed

9 files changed

+122
-46
lines changed

‎ydb/library/actors/interconnect/rdma/cq_actor/cq_actor.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
#include <ydb/library/actors/interconnect/rdma/events.h>
1010
#include <ydb/library/actors/interconnect/rdma/rdma.h>
1111

12+
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
13+
1214
using namespace NActors;
1315

1416
namespace NInterconnect::NRdma {

‎ydb/library/actors/interconnect/rdma/ctx.cpp‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "ctx.h"
1+
#include "ctx_impl.h"
22

33
#include <util/stream/output.h>
44
#include <util/string/builder.h>
@@ -13,18 +13,41 @@ TDeviceCtx::TDeviceCtx(ibv_context* ctx, ibv_pd* pd)
1313
{}
1414

1515
TRdmaCtx::TRdmaCtx(
16-
std::shared_ptr<TDeviceCtx> deviceCtx, ibv_device_attr devAttr, const char* deviceName,
17-
ui32 portNum, ibv_port_attr portAttr, int gidIndex, ibv_gid gid
16+
std::shared_ptr<TDeviceCtx> deviceCtx, const ibv_device_attr& devAttr, const char* deviceName,
17+
ui32 portNum, const ibv_port_attr& portAttr, int gidIndex, const ibv_gid& gid
1818
)
1919
: DeviceCtx(std::move(deviceCtx))
20-
, DevAttr(devAttr)
21-
, DeviceName(deviceName)
22-
, PortNum(portNum)
23-
, PortAttr(portAttr)
24-
, GidIndex(gidIndex)
25-
, Gid(gid)
20+
, Impl(new TImpl(devAttr, deviceName, portNum, portAttr, gidIndex, gid))
2621
{}
2722

23+
const ibv_device_attr& TRdmaCtx::GetDevAttr() const noexcept {
24+
return Impl->DevAttr;
25+
}
26+
27+
const char* TRdmaCtx::GetDeviceName() const noexcept {
28+
return Impl->DeviceName;
29+
}
30+
31+
ui32 TRdmaCtx::GetPortNum() const noexcept {
32+
return Impl->PortNum;
33+
}
34+
35+
const ibv_port_attr& TRdmaCtx::GetPortAttr() const noexcept {
36+
return Impl->PortAttr;
37+
}
38+
39+
int TRdmaCtx::GetGidIndex() const noexcept {
40+
return Impl->GidIndex;
41+
}
42+
43+
const ibv_gid& TRdmaCtx::GetGid() const noexcept {
44+
return Impl->Gid;
45+
}
46+
47+
size_t TRdmaCtx::GetDeviceIndex() const noexcept {
48+
return Impl->DeviceIndex;
49+
}
50+
2851
TDeviceCtx::~TDeviceCtx() {
2952
ibv_dealloc_pd(ProtDomain);
3053
ibv_close_device(Context);

‎ydb/library/actors/interconnect/rdma/ctx.h‎

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@
55
#include <util/system/types.h>
66
#include <util/generic/string.h>
77

8-
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
98

109
extern "C" {
1110

1211
struct ibv_context;
1312
struct ibv_pd;
13+
struct ibv_device_attr;
14+
struct ibv_port_attr;
15+
union ibv_gid;
1416

1517
}
1618

@@ -36,55 +38,38 @@ struct TDeviceCtx : public NNonCopyable::TNonCopyable {
3638
class TRdmaCtx : public NNonCopyable::TNonCopyable {
3739
friend class NLinkMgr::TRdmaLinkManager;
3840
TRdmaCtx(
39-
std::shared_ptr<TDeviceCtx> deviceCtx, ibv_device_attr devAttr, const char* deviceName,
40-
ui32 portNum, ibv_port_attr portAttr, int gidIndex, ibv_gid gid
41+
std::shared_ptr<TDeviceCtx> deviceCtx, const ibv_device_attr& devAttr, const char* deviceName,
42+
ui32 portNum, const ibv_port_attr& portAttr, int gidIndex, const ibv_gid& gid
4143
);
4244

45+
struct TImpl;
4346
public:
4447
static std::shared_ptr<TRdmaCtx> Create(std::shared_ptr<TDeviceCtx> deviceCtx, ui32 portNum, int gidIndex);
4548

4649
~TRdmaCtx() = default;
4750

48-
ibv_context* GetContext() const {
51+
ibv_context* GetContext() const noexcept {
4952
return DeviceCtx->Context;
5053
}
51-
ibv_pd* GetProtDomain() const {
54+
55+
ibv_pd* GetProtDomain() const noexcept {
5256
return DeviceCtx->ProtDomain;
5357
}
54-
const ibv_device_attr& GetDevAttr() const {
55-
return DevAttr;
56-
}
57-
const char* GetDeviceName() const {
58-
return DeviceName;
59-
}
60-
ui32 GetPortNum() const {
61-
return PortNum;
62-
}
63-
const ibv_port_attr& GetPortAttr() const {
64-
return PortAttr;
65-
}
66-
int GetGidIndex() const {
67-
return GidIndex;
68-
}
69-
const ibv_gid& GetGid() const {
70-
return Gid;
71-
}
72-
size_t GetDeviceIndex() const {
73-
return DeviceIndex;
74-
}
58+
59+
const ibv_device_attr& GetDevAttr() const noexcept;
60+
const char* GetDeviceName() const noexcept;
61+
ui32 GetPortNum() const noexcept;
62+
const ibv_port_attr& GetPortAttr() const noexcept;
63+
int GetGidIndex() const noexcept;
64+
const ibv_gid& GetGid() const noexcept;
65+
size_t GetDeviceIndex() const noexcept;
7566

7667
void Output(IOutputStream &str) const;
7768
TString ToString() const;
7869

7970
private:
8071
const std::shared_ptr<TDeviceCtx> DeviceCtx;
81-
const ibv_device_attr DevAttr;
82-
const char* DeviceName;
83-
const ui32 PortNum;
84-
const ibv_port_attr PortAttr;
85-
const int GidIndex;
86-
const ibv_gid Gid;
87-
size_t DeviceIndex;
72+
std::unique_ptr<TImpl> Impl;
8873
};
8974

9075
}

‎ydb/library/actors/interconnect/rdma/ctx_impl.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#pragma once
2+
#include "ctx.h"
3+
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
4+
5+
namespace NInterconnect::NRdma {
6+
7+
struct TRdmaCtx::TImpl {
8+
TImpl(const ibv_device_attr& devAttr, const char* deviceName,
9+
ui32 portNum, const ibv_port_attr& portAttr, int gidIndex, const ibv_gid& gid) noexcept
10+
: DevAttr(devAttr)
11+
, DeviceName(deviceName)
12+
, PortNum(portNum)
13+
, PortAttr(portAttr)
14+
, GidIndex(gidIndex)
15+
, Gid(gid)
16+
{}
17+
18+
const ibv_device_attr DevAttr;
19+
const char* DeviceName;
20+
const ui32 PortNum;
21+
const ibv_port_attr PortAttr;
22+
const int GidIndex;
23+
const ibv_gid Gid;
24+
size_t DeviceIndex;
25+
};
26+
27+
}

‎ydb/library/actors/interconnect/rdma/link_manager.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#include "link_manager.h"
22
#include "ctx.h"
3-
4-
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
3+
#include "ctx_impl.h"
54

65
#include <util/generic/scope.h>
76
#include <util/generic/string.h>
@@ -123,7 +122,7 @@ static class TRdmaLinkManager {
123122
// check for duplicates
124123
for (size_t i = 0; i < CtxMap.size(); ++i) {
125124
auto ctx = CtxMap[i].second;
126-
ctx->DeviceIndex = i;
125+
ctx->Impl->DeviceIndex = i;
127126

128127
if (i > 0) {
129128
auto prevCtx = CtxMap[i - 1].second;

‎ydb/library/actors/interconnect/rdma/link_manager.h‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,32 @@
11
#pragma once
22

3+
#include <util/system/types.h>
4+
5+
#if defined(_linux_)
6+
37
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
48

5-
#include <util/generic/fwd.h>
9+
#else
10+
11+
#ifdef __cplusplus
12+
extern "C" {
13+
#endif
14+
15+
union ibv_gid {
16+
ui8 raw[16];
17+
struct {
18+
ui64 subnet_prefix;
19+
ui64 interface_id;
20+
} global;
21+
};
622

23+
#ifdef __cplusplus
24+
}
25+
#endif
26+
27+
#endif
28+
29+
#include <util/generic/fwd.h>
730

831
struct in6_addr;
932

‎ydb/library/actors/interconnect/rdma/mem_pool.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,21 @@
33

44
#ifndef MEM_POOL_DISABLE_RDMA_SUPPORT
55
#include "ctx.h"
6+
#else
7+
extern "C" {
8+
9+
struct ibv_mr {
10+
struct ibv_context *context;
11+
struct ibv_pd *pd;
12+
void *addr;
13+
size_t length;
14+
ui32 handle;
15+
ui32 lkey;
16+
ui32 rkey;
17+
};
18+
19+
}
20+
621
#endif
722

823
#include <library/cpp/monlib/dynamic_counters/counters.h>

‎ydb/library/actors/interconnect/rdma/rdma_impl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "events.h"
55
#include "rdma.h"
66

7+
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
78
#include <ydb/library/actors/core/actorsystem.h>
89
#include <library/cpp/monlib/metrics/metric_registry.h>
910
#include <library/cpp/monlib/metrics/metric_sub_registry.h>

‎ydb/library/actors/interconnect/rdma/ut/rdma_low_ut.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <string.h>
44

5+
#include <contrib/libs/ibdrv/include/infiniband/verbs.h>
56
#include <ydb/library/actors/interconnect/rdma/ctx.h>
67
#include <ydb/library/actors/interconnect/rdma/events.h>
78
#include <ydb/library/actors/interconnect/rdma/rdma.h>

0 commit comments

Comments
 (0)