Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 34 additions & 4 deletions ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1611,9 +1611,10 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
UNIT_ASSERT(ConvertIPartsToString(parts.Get()) == res->Data.ToString().Slice());
}
}

Y_UNIT_TEST(ChunkWriteDifferentOffsetAndSize) {
for (ui32 i = 0; i <= 3; ++i) {
ChunkWriteDifferentOffsetAndSizeImpl(i & 1, i & 2);
for (ui32 i = 0; i <= 1; ++i) {
ChunkWriteDifferentOffsetAndSizeImpl(i & 1, false);
}
}

Expand Down Expand Up @@ -3035,7 +3036,17 @@ Y_UNIT_TEST_SUITE(TPDiskPrefailureDiskTest) {
pdt.Run();
}
}
/*

// RDMA use ibverbs shared library which is part of hardware vendor provided drivers and can't be linked staticaly
// This library is not MSan-instrumented, so it causes msan fail if we run. So skip such run...
static bool IsMsanEnabled() {
#if defined(_msan_enabled_)
return true;
#else
return false;
#endif
}

Y_UNIT_TEST_SUITE(RDMA) {
void TestChunkReadWithRdmaAllocator(bool plainDataChunks) {
TActorTestContext testCtx({
Expand Down Expand Up @@ -3076,13 +3087,23 @@ Y_UNIT_TEST_SUITE(RDMA) {
}

Y_UNIT_TEST(TestChunkReadWithRdmaAllocatorEncryptedChunks) {
if (IsMsanEnabled())
return;

TestChunkReadWithRdmaAllocator(false);
}

Y_UNIT_TEST(TestChunkReadWithRdmaAllocatorPlainChunks) {
if (IsMsanEnabled())
return;

TestChunkReadWithRdmaAllocator(true);
}

Y_UNIT_TEST(TestRcBuf) {
if (IsMsanEnabled())
return;

ui32 size = 129961;
ui32 offset = 123;
ui32 tailRoom = 1111;
Expand Down Expand Up @@ -3116,6 +3137,15 @@ Y_UNIT_TEST_SUITE(RDMA) {
buf1.RawDataPtr(0, totalSize);
buf2.RawDataPtr(0, totalSize);
}

Y_UNIT_TEST(ChunkWriteDifferentOffsetAndSize) {
if (IsMsanEnabled())
return;

for (ui32 i = 0; i <= 1; ++i) {
NTestSuiteTPDiskTest::ChunkWriteDifferentOffsetAndSizeImpl(i & 1, true);
}
}
}
*/

} // namespace NKikimr
17 changes: 15 additions & 2 deletions ydb/library/actors/interconnect/rdma/link_manager.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "link_manager.h"
#include "ctx.h"
#include "ctx_impl.h"
#include <mutex>

#include <util/generic/scope.h>
#include <util/generic/string.h>
Expand Down Expand Up @@ -60,10 +61,12 @@ static class TRdmaLinkManager {
return;
}
}
private:
TCtxsMap CtxMap;

void ScanDevices() {
std::lock_guard<std::mutex> lock(Mtx);
if (Inited) {
return;
}
int numDevices = 0;
int err;
ibv_device** deviceList = ibv_get_device_list(&numDevices);
Expand Down Expand Up @@ -130,10 +133,16 @@ static class TRdmaLinkManager {
}
}
}
Inited = true;
}

private:
TCtxsMap CtxMap;
int ErrNo = 0;
TString Err;
std::mutex Mtx;
bool Inited = false;


} RdmaLinkManager;

Expand All @@ -159,4 +168,8 @@ const TCtxsMap& GetAllCtxs() {
return RdmaLinkManager.GetAllCtxs();
}

void Init() {
RdmaLinkManager.ScanDevices();
}

}
2 changes: 1 addition & 1 deletion ydb/library/actors/interconnect/rdma/link_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,5 @@ using TCtxsMap = std::vector<std::pair<ibv_gid, std::shared_ptr<NInterconnect::N
TRdmaCtx* GetCtx(int sockfd);
TRdmaCtx* GetCtx(const in6_addr& );
const TCtxsMap& GetAllCtxs();

void Init();
}
7 changes: 6 additions & 1 deletion ydb/library/actors/interconnect/rdma/mem_pool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -316,10 +316,15 @@ namespace NInterconnect::NRdma {
return counters;
}

static const NInterconnect::NRdma::NLinkMgr::TCtxsMap& GetAllCtxs() {
NInterconnect::NRdma::NLinkMgr::Init();
return NInterconnect::NRdma::NLinkMgr::GetAllCtxs();
}

class TMemPoolBase: public IMemPool {
public:
TMemPoolBase(size_t maxChunk, NMonitoring::TDynamicCounterPtr counter)
: Ctxs(NInterconnect::NRdma::NLinkMgr::GetAllCtxs())
: Ctxs(GetAllCtxs())
, MaxChunk(maxChunk)
, Alignment(NSystemInfo::GetPageSize())
{
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/actors/interconnect/rdma/ya.make
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,6 @@ RECURSE(
cq_actor
)

#RECURSE_FOR_TESTS(
# ut
#)
RECURSE_FOR_TESTS(
ut
)
Loading