[ICRDMA] Do not scan devices from ctor. Restore ut. EXT-1639 (#28087)

Access to ibverbs library from ctor of LinkManager causes issue with msan for whole project. To solve it we will initialise rdma context only if it needed. Msan tests will not be run for rdma related code.

Author: dcherednik

ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp

@@ -1284,9 +1284,10 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
             UNIT_ASSERT(ConvertIPartsToString(parts.Get()) == res->Data.ToString().Slice());
         }
     }
+
     Y_UNIT_TEST(ChunkWriteDifferentOffsetAndSize) {
-        for (ui32 i = 0; i <= 3; ++i) {
-            ChunkWriteDifferentOffsetAndSizeImpl(i & 1, i & 2);
+        for (ui32 i = 0; i <= 1; ++i) {
+            ChunkWriteDifferentOffsetAndSizeImpl(i & 1, false);
         }
     }
 
@@ -2129,7 +2130,17 @@ Y_UNIT_TEST_SUITE(ShredPDisk) {
 #endif
 }
 
-/*
+
+// RDMA use ibverbs shared library which is part of hardware vendor provided drivers and can't be linked staticaly
+// This library is not MSan-instrumented, so it causes msan fail if we run. So skip such run...
+static bool IsMsanEnabled() {
+#if defined(_msan_enabled_)
+    return true;
+#else
+    return false;
+#endif
+}
+
 Y_UNIT_TEST_SUITE(RDMA) {
     void TestChunkReadWithRdmaAllocator(bool plainDataChunks) {
         TActorTestContext testCtx({
@@ -2170,13 +2181,23 @@ Y_UNIT_TEST_SUITE(RDMA) {
     }
 
     Y_UNIT_TEST(TestChunkReadWithRdmaAllocatorEncryptedChunks) {
+        if (IsMsanEnabled())
+            return;
+
         TestChunkReadWithRdmaAllocator(false);
     }
+
     Y_UNIT_TEST(TestChunkReadWithRdmaAllocatorPlainChunks) {
+        if (IsMsanEnabled())
+            return;
+
         TestChunkReadWithRdmaAllocator(true);
     }
 
     Y_UNIT_TEST(TestRcBuf) {
+        if (IsMsanEnabled())
+            return;
+
         ui32 size = 129961;
         ui32 offset = 123;
         ui32 tailRoom = 1111;
@@ -2210,6 +2231,15 @@ Y_UNIT_TEST_SUITE(RDMA) {
         buf1.RawDataPtr(0, totalSize);
         buf2.RawDataPtr(0, totalSize);
     }
+
+    Y_UNIT_TEST(ChunkWriteDifferentOffsetAndSize) {
+        if (IsMsanEnabled())
+            return;
+
+        for (ui32 i = 0; i <= 1; ++i) {
+            NTestSuiteTPDiskTest::ChunkWriteDifferentOffsetAndSizeImpl(i & 1, true);
+        }
+    }
 }
-*/
+
 } // namespace NKikimr

ydb/library/actors/interconnect/rdma/link_manager.cpp

@@ -1,6 +1,7 @@
 #include "link_manager.h"
 #include "ctx.h"
 #include "ctx_impl.h"
+#include <mutex>
 
 #include <util/generic/scope.h>
 #include <util/generic/string.h>
@@ -60,10 +61,12 @@ static class TRdmaLinkManager {
             return;
         }
     }
-private:
-    TCtxsMap CtxMap;
 
     void ScanDevices() {
+        std::lock_guard<std::mutex> lock(Mtx);
+        if (Inited) {
+            return;
+        }
         int numDevices = 0;
         int err;
         ibv_device** deviceList = ibv_get_device_list(&numDevices);
@@ -130,10 +133,16 @@ static class TRdmaLinkManager {
                 }
             }
         }
+        Inited = true;
     }
 
+private:
+    TCtxsMap CtxMap;
     int ErrNo = 0;
     TString Err;
+    std::mutex Mtx;
+    bool Inited = false;
+
 
 } RdmaLinkManager;
 
@@ -159,4 +168,8 @@ const TCtxsMap& GetAllCtxs() {
     return RdmaLinkManager.GetAllCtxs();
 }
 
+void Init() {
+    RdmaLinkManager.ScanDevices();
+}
+
 } 

ydb/library/actors/interconnect/rdma/link_manager.h

@@ -44,5 +44,5 @@ using TCtxsMap = std::vector<std::pair<ibv_gid, std::shared_ptr<NInterconnect::N
 TRdmaCtx* GetCtx(int sockfd);
 TRdmaCtx* GetCtx(const in6_addr& );
 const TCtxsMap& GetAllCtxs();
-
+void Init();
 }

ydb/library/actors/interconnect/rdma/mem_pool.cpp

@@ -277,10 +277,15 @@ namespace NInterconnect::NRdma {
         return counters;
     }
 
+    static const NInterconnect::NRdma::NLinkMgr::TCtxsMap& GetAllCtxs() {
+        NInterconnect::NRdma::NLinkMgr::Init();
+        return NInterconnect::NRdma::NLinkMgr::GetAllCtxs();
+    }
+
     class TMemPoolBase: public IMemPool {
     public:
         TMemPoolBase(size_t maxChunk, NMonitoring::TDynamicCounterPtr counter)
-            : Ctxs(NInterconnect::NRdma::NLinkMgr::GetAllCtxs())
+            : Ctxs(GetAllCtxs())
             , MaxChunk(maxChunk)
             , Alignment(NSystemInfo::GetPageSize())
         {

ydb/library/actors/interconnect/rdma/ya.make

@@ -35,6 +35,6 @@ RECURSE(
     cq_actor
 )
 
-#RECURSE_FOR_TESTS(
-#    ut
-#)
+RECURSE_FOR_TESTS(
+    ut
+)