diff --git a/.github/workflows/.spellcheck-conf.toml b/.github/workflows/.spellcheck-conf.toml index 288af6a19..bb0f480d6 100644 --- a/.github/workflows/.spellcheck-conf.toml +++ b/.github/workflows/.spellcheck-conf.toml @@ -1,6 +1,6 @@ [default] # Don't correct the following words: -extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN"] +extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN", "usm"] [files] # completely exclude those files from consideration: diff --git a/docs/config/api.rst b/docs/config/api.rst index 879623428..a1b07b0ce 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -170,6 +170,19 @@ Memtarget .. doxygenfile:: experimental/memtarget.h :sections: define enum typedef func +Memory Properties +========================================== + +TODO: Add general information about memory properties. + +.. note:: + The memory properties APIs are experimental and may change in future releases. + +Memory Properties +------------------------------------------ +.. doxygenfile:: experimental/memory_props.h + :sections: define enum typedef func var + Inter-Process Communication ========================================== diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt index d4e40a3ec..064feae55 100644 --- a/docs/config/spelling_exceptions.txt +++ b/docs/config/spelling_exceptions.txt @@ -3,22 +3,24 @@ allocatable allocator allocators calloc -CXL copyable +CUcontext +CUdevice customizable +CXL daxX -deallocation deallocating +deallocation deallocations -Devdax dev +Devdax Globals +highPtr hMemtarget hPool hProvider -highPtr -io interprocess +io ipc jemalloc lowPtr @@ -47,8 +49,9 @@ partList pid poolable preallocated -providerIpcData +propertyId providential +providerIpcData ptr realloc Scalable @@ -57,11 +60,15 @@ stdout Tiering tiering topologies +uint +uintptr umf umfGetIPCHandle umfMemoryProviderAlloc umfMemoryProviderGetLastNativeError umfMemoryProviderOpenIPCHandle +umfMemspaceMemtargetAdd +umfMemspaceUserFilter umfOsMemoryProviderParamsDestroy umfPool umfPoolCalloc @@ -69,6 +76,6 @@ umfPoolDestroy umfPoolGetTag umfPoolMallocUsableSize umfPoolRealloc -umfMemspaceUserFilter -umfMemspaceMemtargetAdd -unfreed \ No newline at end of file +unfreed +usm +ze diff --git a/include/umf/base.h b/include/umf/base.h index f7cd9de63..aaba04c57 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -51,6 +51,39 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; +/// @brief Handle to the memory properties structure +typedef struct umf_memory_properties_t *umf_memory_properties_handle_t; + +/// @brief ID of the memory property +typedef enum umf_memory_property_id_t { + UMF_MEMORY_PROPERTY_INVALID = -1, ///< Invalid property + + // UMF specific + UMF_MEMORY_PROPERTY_PROVIDER_HANDLE = + 0, ///< Handle to the memory provider (void*) + UMF_MEMORY_PROPERTY_POOL_HANDLE = 1, ///< Handle to the memory pool (void*) + + // generic pointer properties + UMF_MEMORY_PROPERTY_BASE_ADDRESS = + 10, ///< Base address of the allocation (uintptr_t) + UMF_MEMORY_PROPERTY_BASE_SIZE = + 11, ///< Base size of the allocation (size_t) + UMF_MEMORY_PROPERTY_BUFFER_ID = + 12, ///< Unique identifier for the buffer (uint64_t) + + // GPU specific + UMF_MEMORY_PROPERTY_POINTER_TYPE = + 20, ///< Type of the pointer (umf_usm_memory_type_t) + UMF_MEMORY_PROPERTY_CONTEXT = + 21, ///< GPU context of the allocation (depending on GPU provider, e.g. ze_context_handle_t, CUcontext) + UMF_MEMORY_PROPERTY_DEVICE = + 22, ///< GPU device where the allocation resides (depending on GPU provider, e.g. ze_device_handle_t, CUdevice) + + /// @cond + UMF_MEMORY_PROPERTY_MAX_RESERVED = 0x1000, ///< Maximum reserved value + /// @endcond +} umf_memory_property_id_t; + /// @brief Type of the CTL query typedef enum umf_ctl_query_type { CTL_QUERY_READ, diff --git a/include/umf/experimental/memory_props.h b/include/umf/experimental/memory_props.h new file mode 100644 index 000000000..33fac315b --- /dev/null +++ b/include/umf/experimental/memory_props.h @@ -0,0 +1,46 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPS_H +#define UMF_MEMORY_PROPS_H 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @brief Get the memory properties handle for a given pointer +/// \details +/// The handle returned by this function is valid until the memory pointed +/// to by the pointer is freed. +/// @param ptr pointer to the allocated memory +/// @param props_handle [out] pointer to the memory properties handle +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle); + +/// @brief Get a specific memory property from the properties handle +/// @param props_handle handle to the memory properties +/// @param memory_property_id ID of the memory property to get +/// @param max_property_size size of the property value buffer +/// @param property_value [out] pointer to the value of the memory property +/// which will be filled +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t max_property_size, + void *property_value); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_MEMORY_PROPS_H */ diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 16e1536fd..f70b7ed59 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -278,6 +278,21 @@ typedef struct umf_memory_provider_ops_t { const char *name, void *arg, size_t size, umf_ctl_query_type_t queryType, va_list args); + /// + /// @brief Retrieve properties of the memory allocation. + /// @param provider pointer to the memory provider + /// @param ptr pointer to the allocated memory + /// @param memory_property_id identifier of the memory property to retrieve + /// @param property_value [out] pointer to the value of the memory property + /// which will be filled + /// @param max_property_size size of the property value buffer + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure + /// + umf_result_t (*ext_get_allocation_properties)( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *property_value, + size_t max_property_size); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 990a75dc2..c25fee6ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,6 +42,7 @@ set(UMF_SOURCES ipc.c ipc_cache.c memory_pool.c + memory_props.c memory_provider.c memory_provider_get_last_failed.c memtarget.c diff --git a/src/ipc.c b/src/ipc.c index d4e5cc806..29ed5bac2 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -58,14 +58,19 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } size_t ipcHandleSize = 0; - umf_alloc_info_t allocInfo; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("cannot get alloc info for ptr = %p.", ptr); + LOG_ERR("cannot get alloc props for ptr = %p.", ptr); return ret; } - ret = umfPoolGetIPCHandleSize(allocInfo.pool, &ipcHandleSize); + if (props == NULL || props->pool == NULL) { + LOG_ERR("cannot get pool from alloc info for ptr = %p.", ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + ret = umfPoolGetIPCHandleSize(props->pool, &ipcHandleSize); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("cannot get IPC handle size."); return ret; @@ -79,11 +84,14 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, // We cannot use umfPoolGetMemoryProvider function because it returns // upstream provider but we need tracking one - umf_memory_provider_handle_t provider = allocInfo.pool->provider; - assert(provider); + if (props->pool->provider == NULL) { + LOG_ERR("cannot get memory provider from pool"); + umf_ba_global_free(ipcData); + return UMF_RESULT_ERROR_UNKNOWN; + } + umf_memory_provider_handle_t provider = props->pool->provider; - ret = umfMemoryProviderGetIPCHandle(provider, allocInfo.base, - allocInfo.baseSize, + ret = umfMemoryProviderGetIPCHandle(provider, props->base, props->base_size, (void *)ipcData->providerIpcData); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to get IPC handle."); @@ -92,10 +100,10 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } // ipcData->handle_id is filled by tracking provider - ipcData->base = allocInfo.base; + ipcData->base = props->base; ipcData->pid = utils_getpid(); - ipcData->baseSize = allocInfo.baseSize; - ipcData->offset = (uintptr_t)ptr - (uintptr_t)allocInfo.base; + ipcData->baseSize = props->base_size; + ipcData->offset = (uintptr_t)ptr - (uintptr_t)props->base; *umfIPCHandle = ipcData; *size = ipcHandleSize; diff --git a/src/libumf.def b/src/libumf.def index 0159ddbe2..a1080197a 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -144,3 +144,6 @@ EXPORTS umfJemallocPoolParamsDestroy umfJemallocPoolParamsSetNumArenas umfPoolGetName +; Added in UMF_1.1 + umfGetMemoryPropertiesHandle + umfGetMemoryProperty diff --git a/src/libumf.map b/src/libumf.map index 348675ff0..ccc369493 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -141,3 +141,8 @@ UMF_1.0 { local: *; }; + +UMF_1.1 { + umfGetMemoryPropertiesHandle; + umfGetMemoryProperty; +} UMF_1.0; diff --git a/src/memory_pool.c b/src/memory_pool.c index 004f42d9e..dce7559f8 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -323,9 +323,18 @@ umf_result_t umfFree(void *ptr) { } umf_result_t umfPoolByPtr(const void *ptr, umf_memory_pool_handle_t *pool) { - UMF_CHECK((pool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - *pool = umfMemoryTrackerGetPool(ptr); - return *pool ? UMF_RESULT_SUCCESS : UMF_RESULT_ERROR_INVALID_ARGUMENT; + UMF_CHECK(pool != NULL, UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK(ptr != NULL, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); + if (ret != UMF_RESULT_SUCCESS || props == NULL || props->pool == NULL) { + *pool = NULL; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *pool = props->pool; + return UMF_RESULT_SUCCESS; } umf_result_t umfPoolGetMemoryProvider(umf_memory_pool_handle_t hPool, diff --git a/src/memory_props.c b/src/memory_props.c new file mode 100644 index 000000000..0ec63d0cb --- /dev/null +++ b/src/memory_props.c @@ -0,0 +1,127 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include +#include +#include + +#include "memory_props_internal.h" +#include "memory_provider_internal.h" +#include "provider/provider_tracking.h" + +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle) { + UMF_CHECK((props_handle != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + tracker_alloc_info_t *info = NULL; + umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &info); + if (ret == UMF_RESULT_SUCCESS) { + *props_handle = &info->props; + return UMF_RESULT_SUCCESS; + } + + // try to get IPC info + umf_ipc_info_t ipc_info; + ret = umfMemoryTrackerGetIpcInfo(ptr, &ipc_info); + if (ret == UMF_RESULT_SUCCESS) { + *props_handle = ipc_info.props; + return UMF_RESULT_SUCCESS; + } + + LOG_ERR("Failed to get memory properties handle for ptr=%p", ptr); + return ret; +} + +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t max_property_size, void *value) { + UMF_CHECK((value != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((props_handle != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((max_property_size > 0), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_memory_provider_t *provider = props_handle->provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_INVALID: + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + + case UMF_MEMORY_PROPERTY_POOL_HANDLE: + if (UNLIKELY(max_property_size < sizeof(umf_memory_pool_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_memory_pool_handle_t *)value = props_handle->pool; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_PROVIDER_HANDLE: + if (UNLIKELY(max_property_size < + sizeof(umf_memory_provider_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_memory_provider_handle_t *)value = provider; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BUFFER_ID: + if (UNLIKELY(max_property_size < sizeof(uint64_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(uint64_t *)value = props_handle->id; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_ADDRESS: + if (UNLIKELY(max_property_size < sizeof(uintptr_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(uintptr_t *)value = (uintptr_t)props_handle->base; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_SIZE: + if (UNLIKELY(max_property_size < sizeof(size_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(size_t *)value = props_handle->base_size; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + // NOTE: this property is "cached" in the props_handle but the value is + // determined by the memory provider and set during addition to the + // tracker. + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = props_handle->memory_type; + return UMF_RESULT_SUCCESS; + + // GPU Memory Provider specific properties + case UMF_MEMORY_PROPERTY_CONTEXT: + case UMF_MEMORY_PROPERTY_DEVICE: + return provider->ops.ext_get_allocation_properties( + provider->provider_priv, props_handle->base, memory_property_id, + value, max_property_size); + + default: + break; + }; + + if (memory_property_id > UMF_MEMORY_PROPERTY_MAX_RESERVED) { + // custom memory properties should be handled by the user provider + if (provider->ops.ext_get_allocation_properties) { + return provider->ops.ext_get_allocation_properties( + provider->provider_priv, props_handle->base, memory_property_id, + value, max_property_size); + } + + LOG_ERR("Unknown memory property ID: %d", memory_property_id); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} diff --git a/src/memory_props_internal.h b/src/memory_props_internal.h new file mode 100644 index 000000000..fc52c42ce --- /dev/null +++ b/src/memory_props_internal.h @@ -0,0 +1,42 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPS_INTERNAL_H +#define UMF_MEMORY_PROPS_INTERNAL_H 1 + +#include + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct umf_memory_properties_t { + umf_memory_pool_handle_t pool; + umf_memory_provider_handle_t provider; + uint64_t id; + void *base; + size_t base_size; + umf_usm_memory_type_t memory_type; +} umf_memory_properties_t; + +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, void *property_value, + size_t max_property_size); + +#ifdef __cplusplus +} +#endif + +#endif // UMF_MEMORY_PROPS_INTERNAL_H diff --git a/src/memory_provider.c b/src/memory_provider.c index 8ed2a79db..b6391a98c 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -133,6 +133,17 @@ umfDefaultCtlHandle(void *provider, umf_ctl_query_source_t operationType, return UMF_RESULT_ERROR_NOT_SUPPORTED; } +static umf_result_t umfDefaultGetAllocationProperties( + void *provider, const void *ptr, umf_memory_property_id_t propertyId, + void *property_value, size_t max_property_size) { + (void)provider; + (void)ptr; + (void)propertyId; + (void)property_value; + (void)max_property_size; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_purge_lazy) { ops->ext_purge_lazy = umfDefaultPurgeLazy; @@ -153,6 +164,10 @@ void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_ctl) { ops->ext_ctl = umfDefaultCtlHandle; } + + if (!ops->ext_get_allocation_properties) { + ops->ext_get_allocation_properties = umfDefaultGetAllocationProperties; + } } void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { @@ -499,3 +514,26 @@ umfMemoryProviderCloseIPCHandle(umf_memory_provider_handle_t hProvider, checkErrorAndSetLastProvider(res, hProvider); return res; } + +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, void *property_value, + size_t max_property_size) { + + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((property_value != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((max_property_size != 0), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((propertyId != UMF_MEMORY_PROPERTY_INVALID), + UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // NOTE: we do not check if the propertyId is below + // UMF_MEMORY_PROPERTY_MAX_RESERVED value, as the user could use a custom + // property ID that is above the reserved range + + umf_result_t res = hProvider->ops.ext_get_allocation_properties( + hProvider->provider_priv, ptr, propertyId, property_value, + max_property_size); + + checkErrorAndSetLastProvider(res, hProvider); + return res; +} diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index e5339376e..608e1b38a 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -982,14 +982,19 @@ umf_result_t disjoint_pool_malloc_usable_size(void *pool, const void *ptr, critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - *size = 0; - return ret; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } - *size = allocInfo.baseSize; + *size = props->base_size; return UMF_RESULT_SUCCESS; } @@ -1025,15 +1030,21 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { TLS_last_allocation_error = ret; LOG_ERR("failed to get allocation info from the memory tracker"); return ret; } - size_t size = allocInfo.baseSize; + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + size_t size = props->base_size; umf_memory_provider_handle_t provider = disjoint_pool->provider; ret = umfMemoryProviderFree(provider, ptr, size); if (ret != UMF_RESULT_SUCCESS) { diff --git a/src/pool/pool_proxy.c b/src/pool/pool_proxy.c index c6bf74124..3da9a4ee6 100644 --- a/src/pool/pool_proxy.c +++ b/src/pool/pool_proxy.c @@ -100,11 +100,22 @@ static umf_result_t proxy_free(void *pool, void *ptr) { struct proxy_memory_pool *hPool = (struct proxy_memory_pool *)pool; if (ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t umf_result = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (umf_result == UMF_RESULT_SUCCESS) { - size = allocInfo.baseSize; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + + if (umf_result != UMF_RESULT_SUCCESS) { + TLS_last_allocation_error = umf_result; + LOG_ERR("failed to get allocation info from the memory tracker"); + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } + + size = props->base_size; } return umfMemoryProviderFree(hPool->hProvider, ptr, size); diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 983be6b55..b721f3d47 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -12,6 +12,7 @@ #include #include +#include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_cuda_internal.h" #include "utils_load_library.h" @@ -712,6 +713,45 @@ static umf_result_t cu_ctl(void *provider, umf_ctl_query_source_t operationType, query_type, arg, size, args); } +static umf_result_t cu_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *value, + size_t max_property_size) { + + // unused + (void)ptr; + + cu_memory_provider_t *cuda_provider = (cu_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = cuda_provider->memory_type; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + if (UNLIKELY(max_property_size < sizeof(CUcontext))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(CUcontext *)value = cuda_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + if (UNLIKELY(max_property_size < sizeof(CUdevice))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(CUdevice *)value = cuda_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + }; + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -735,6 +775,8 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = cu_memory_provider_open_ipc_handle, .ext_close_ipc_handle = cu_memory_provider_close_ipc_handle, .ext_ctl = cu_ctl, + .ext_get_allocation_properties = + cu_memory_provider_get_allocation_properties, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index 5cf1ac0fd..f56ffac80 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -586,7 +586,9 @@ static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = devdax_put_ipc_handle, .ext_open_ipc_handle = devdax_open_ipc_handle, .ext_close_ipc_handle = devdax_close_ipc_handle, - .ext_ctl = devdax_ctl}; + .ext_ctl = devdax_ctl, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfDevDaxMemoryProviderOps(void) { return &UMF_DEVDAX_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 1031993d2..67add5872 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -913,7 +913,9 @@ static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = file_put_ipc_handle, .ext_open_ipc_handle = file_open_ipc_handle, .ext_close_ipc_handle = file_close_ipc_handle, - .ext_ctl = file_ctl}; + .ext_ctl = file_ctl, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfFileMemoryProviderOps(void) { return &UMF_FILE_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_fixed_memory.c b/src/provider/provider_fixed_memory.c index 08fd3e7f6..10b48b5fc 100644 --- a/src/provider/provider_fixed_memory.c +++ b/src/provider/provider_fixed_memory.c @@ -311,7 +311,9 @@ static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = NULL, .ext_open_ipc_handle = NULL, .ext_close_ipc_handle = NULL, - .ext_ctl = fixed_ctl}; + .ext_ctl = fixed_ctl, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) { return &UMF_FIXED_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index d5e79b244..23127782f 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -14,6 +14,7 @@ #include #include +#include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_level_zero_internal.h" #include "utils_load_library.h" @@ -32,6 +33,7 @@ void fini_ze_global_state(void) { #include "base_alloc_global.h" #include "libumf.h" +#include "provider_level_zero_internal.h" #include "utils_assert.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -838,6 +840,45 @@ static umf_result_t ze_ctl(void *hProvider, query_type, arg, size, args); } +static umf_result_t ze_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *value, + size_t max_property_size) { + (void)ptr; + + struct ze_memory_provider_t *ze_provider = + (struct ze_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = + ze2umf_memory_type(ze_provider->memory_type); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + if (UNLIKELY(max_property_size < sizeof(ze_context_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(ze_context_handle_t *)value = ze_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + if (UNLIKELY(max_property_size < sizeof(ze_device_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(ze_device_handle_t *)value = ze_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + } + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = ze_memory_provider_initialize, @@ -858,6 +899,8 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = ze_memory_provider_open_ipc_handle, .ext_close_ipc_handle = ze_memory_provider_close_ipc_handle, .ext_ctl = ze_ctl, + .ext_get_allocation_properties = + ze_memory_provider_get_allocation_properties, }; const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 659f90841..74c78005b 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1339,7 +1339,8 @@ static umf_result_t os_open_ipc_handle(void *provider, void *providerIpcData, os_ipc_data->visibility, fd, os_ipc_data->fd_offset); if (*ptr == NULL) { os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, errno); - LOG_PERR("memory mapping failed"); + LOG_PERR("memory mapping failed: %zu bytes at fd=%d, offset=%zu", + os_ipc_data->size, fd, os_ipc_data->fd_offset); ret = UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } @@ -1398,6 +1399,7 @@ static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = os_open_ipc_handle, .ext_close_ipc_handle = os_close_ipc_handle, .ext_ctl = os_ctl, + .ext_get_allocation_properties = NULL, }; const umf_memory_provider_ops_t *umfOsMemoryProviderOps(void) { diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 386eef0ba..cf7d812a2 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -15,6 +15,7 @@ #include #include +#include #include #include "base_alloc_global.h" @@ -22,6 +23,8 @@ #include "ipc_cache.h" #include "ipc_internal.h" #include "memory_pool_internal.h" +#include "memory_props_internal.h" +#include "memory_provider_internal.h" #include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -32,6 +35,8 @@ uint64_t IPC_HANDLE_ID = 0; +uint64_t unique_alloc_id = 0; // requires atomic access + struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; // Multilevel maps are needed to support the case @@ -43,19 +48,8 @@ struct umf_memory_tracker_t { critnib *ipc_segments_map; }; -typedef struct tracker_alloc_info_t { - umf_memory_pool_handle_t pool; - size_t size; - // number of overlapping memory regions - // in the next level of map - // falling within the current range - size_t n_children; -#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) - uint64_t is_freed; -#endif -} tracker_alloc_info_t; - typedef struct tracker_ipc_info_t { + umf_memory_properties_t props; size_t size; umf_memory_provider_handle_t provider; ipc_opened_cache_value_t *ipc_cache_value; @@ -113,7 +107,8 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && ((uintptr_t)ptr < rkey + rsize) && n_children) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -192,8 +187,26 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - value->pool = pool; - value->size = size; + // get provider + umf_memory_provider_handle_t provider = NULL; + umfPoolGetMemoryProvider(pool, &provider); + + // get memory type + umf_usm_memory_type_t memory_type = UMF_MEMORY_TYPE_UNKNOWN; + if (provider && provider->ops.ext_get_allocation_properties) { + provider->ops.ext_get_allocation_properties( + provider->provider_priv, ptr, UMF_MEMORY_PROPERTY_POINTER_TYPE, + &memory_type, sizeof(umf_usm_memory_type_t)); + } + + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = pool; + value->props.provider = provider; + value->props.memory_type = memory_type; + value->n_children = 0; #if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) value->is_freed = 0; @@ -214,8 +227,8 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, "child #%zu added to memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", n_children, (void *)hTracker, level - 1, - (void *)parent_value->pool, (void *)parent_key, - parent_value->size); + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); critnib_release(hTracker->alloc_segments_map[level - 1], ref_parent_value); @@ -286,7 +299,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, assert(is_freed != 0xDEADBEEF); #endif - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); if ((uintptr_t)ptr < rkey + rsize) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -300,8 +314,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, "cannot insert to the tracker value (pool=%p, ptr=%p, " "size=%zu) " "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", - (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, - (size_t)rsize); + (void *)pool, ptr, size, (void *)rvalue->props.pool, + (void *)rkey, (size_t)rsize); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } parent_key = rkey; @@ -363,7 +377,8 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " "size=%zu", - (void *)hTracker, level, (void *)value->pool, ptr, value->size); + (void *)hTracker, level, (void *)value->props.pool, ptr, + value->props.base_size); // release the reference to the value got from critnib_remove() assert(ref_value); @@ -375,8 +390,9 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG( "child #%zu removed from memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", - n_children, (void *)hTracker, level - 1, (void *)parent_value->pool, - (void *)parent_key, parent_value->size); + n_children, (void *)hTracker, level - 1, + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); assert(level >= 1); @@ -405,10 +421,26 @@ umfMemoryTrackerAddIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + // get memory type + umf_usm_memory_type_t memory_type = UMF_MEMORY_TYPE_UNKNOWN; + if (provider && provider->ops.ext_get_allocation_properties) { + provider->ops.ext_get_allocation_properties( + provider->provider_priv, ptr, UMF_MEMORY_PROPERTY_POINTER_TYPE, + &memory_type, sizeof(umf_usm_memory_type_t)); + } + value->size = size; value->provider = provider; value->ipc_cache_value = cache_entry; + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = NULL; // unknown + value->props.provider = provider; + value->props.memory_type = memory_type; + int ret = critnib_insert(hTracker->ipc_segments_map, (uintptr_t)ptr, value, 0); if (ret == 0) { @@ -458,37 +490,26 @@ umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_SUCCESS; } -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - return NULL; - } - - return allocInfo.pool; -} - umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo) { - assert(pAllocInfo); + tracker_alloc_info_t **info) { + assert(info); - if (ptr == NULL) { + if (UNLIKELY(ptr == NULL)) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (TRACKER == NULL) { + if (UNLIKELY(TRACKER == NULL)) { LOG_ERR("tracker does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map[0] == NULL) { + if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } tracker_alloc_info_t *top_most_value = NULL; tracker_alloc_info_t *rvalue = NULL; - uintptr_t top_most_key = 0; uintptr_t rkey = 0; uint64_t rsize = 0; size_t n_children = 0; @@ -514,7 +535,6 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, critnib_release(TRACKER->alloc_segments_map[level], ref_value); } top_most_value = NULL; - top_most_key = 0; rkey = 0; rsize = 0; level = 0; @@ -525,10 +545,10 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && (uintptr_t)ptr < rkey + rsize) { - top_most_key = rkey; top_most_value = rvalue; if (ref_top_most_value) { assert(level >= 1); @@ -555,9 +575,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)top_most_key; - pAllocInfo->baseSize = top_most_value->size; - pAllocInfo->pool = top_most_value->pool; + *info = top_most_value; assert(ref_top_most_value); critnib_release(TRACKER->alloc_segments_map[ref_level], ref_top_most_value); @@ -603,6 +621,8 @@ umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, pIpcInfo->baseSize = rvalue->size; pIpcInfo->provider = rvalue->provider; + pIpcInfo->props = &rvalue->props; + if (ref_value) { critnib_release(TRACKER->ipc_segments_map, ref_value); } @@ -692,9 +712,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } - if (value->size != totalSize) { + if (value->props.base_size != totalSize) { LOG_ERR("tracked size=%zu does not match requested size to split: %zu", - value->size, totalSize); + value->props.base_size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } @@ -732,7 +752,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, } // update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&value->size, firstSize); + utils_atomic_store_release_u64((uint64_t *)&value->props.base_size, + firstSize); critnib_release(provider->hTracker->alloc_segments_map[level], ref_value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); @@ -805,12 +826,12 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->pool != highValue->pool) { + if (lowValue->props.pool != highValue->props.pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->size + highValue->size != totalSize) { + if (lowValue->props.base_size + highValue->props.base_size != totalSize) { LOG_FATAL("lowValue->size + highValue->size != totalSize"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; @@ -824,7 +845,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, } // we only need to update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&lowValue->size, totalSize); + utils_atomic_store_release_u64((uint64_t *)&lowValue->props.base_size, + totalSize); size_t low_children = lowValue->n_children; size_t high_children = highValue->n_children; @@ -950,12 +972,13 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, FIND_G, &rkey, (void **)&rvalue, &ref_value)) { - if (rvalue && ((rvalue->pool == pool) || pool == NULL)) { + if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { n_items++; LOG_DEBUG( "found abandoned allocation in the tracking provider: " "pool=%p, ptr=%p, size=%zu", - (void *)rvalue->pool, (void *)rkey, (size_t)rvalue->size); + (void *)rvalue->props.pool, (void *)rkey, + (size_t)rvalue->props.base_size); } if (ref_value) { @@ -1295,6 +1318,16 @@ static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, return umf_result; } +static umf_result_t +trackingGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, + void *value, size_t max_property_size) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetAllocationProperties( + p->hUpstream, ptr, memory_property_id, value, max_property_size); +} + umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = trackingInitialize, @@ -1313,7 +1346,10 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_get_ipc_handle = trackingGetIpcHandle, .ext_put_ipc_handle = trackingPutIpcHandle, .ext_open_ipc_handle = trackingOpenIpcHandle, - .ext_close_ipc_handle = trackingCloseIpcHandle}; + .ext_close_ipc_handle = trackingCloseIpcHandle, + .ext_ctl = NULL, + .ext_get_allocation_properties = trackingGetAllocationProperties, +}; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { (void)unused; diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index d7ee06c1b..9b6884928 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -20,6 +20,7 @@ #include "base_alloc.h" #include "critnib.h" +#include "memory_props_internal.h" #include "utils_concurrency.h" #ifdef __cplusplus @@ -34,18 +35,23 @@ extern umf_memory_tracker_handle_t TRACKER; umf_result_t umfMemoryTrackerCreate(umf_memory_tracker_handle_t *handle); void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle); -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr); +typedef struct tracker_alloc_info_t { + umf_memory_properties_t props; -typedef struct umf_alloc_info_t { - void *base; - size_t baseSize; - umf_memory_pool_handle_t pool; -} umf_alloc_info_t; + // number of overlapping memory regions in the next level of map falling + // within the current range + size_t n_children; +#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) + uint64_t is_freed; +#endif +} tracker_alloc_info_t; umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo); + tracker_alloc_info_t **info); typedef struct umf_ipc_info_t { + umf_memory_properties_handle_t props; + void *base; size_t baseSize; umf_memory_provider_handle_t provider; diff --git a/src/utils/utils_level_zero.cpp b/src/utils/utils_level_zero.cpp index 6daab3e69..e2b62064c 100644 --- a/src/utils/utils_level_zero.cpp +++ b/src/utils/utils_level_zero.cpp @@ -752,6 +752,18 @@ ze_memory_type_t utils_ze_get_mem_type(ze_context_handle_t context, void *ptr) { return alloc_props.type; } +void utils_ze_get_mem_props(ze_context_handle_t context, void *ptr, + ze_memory_allocation_properties_t *alloc_props, + ze_device_handle_t *device) { + alloc_props->stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + alloc_props->pNext = NULL; + alloc_props->type = ZE_MEMORY_TYPE_UNKNOWN; + alloc_props->id = 0; + alloc_props->pageSize = 0; + + libze_ops.zeMemGetAllocProperties(context, ptr, alloc_props, device); +} + int64_t utils_ze_get_num_memory_properties(ze_device_handle_t device) { uint32_t pCount = 0; ze_result_t ze_result = diff --git a/src/utils/utils_level_zero.h b/src/utils/utils_level_zero.h index 00f55b351..41e183288 100644 --- a/src/utils/utils_level_zero.h +++ b/src/utils/utils_level_zero.h @@ -44,6 +44,10 @@ int utils_ze_destroy_context(ze_context_handle_t context); ze_memory_type_t utils_ze_get_mem_type(ze_context_handle_t context, void *ptr); +void utils_ze_get_mem_props(ze_context_handle_t context, void *ptr, + ze_memory_allocation_properties_t *alloc_props, + ze_device_handle_t *device); + int64_t utils_ze_get_num_memory_properties(ze_device_handle_t device); #ifdef __cplusplus diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 753cea02a..ce244ca1d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -55,11 +55,13 @@ function(build_umf_test) if(UMF_CUDA_ENABLED) set(INC_DIRS ${INC_DIRS} ${CUDA_INCLUDE_DIRS}) set(LIB_DIRS ${LIB_DIRS} ${CUDA_LIBRARY_DIRS}) + set(CPL_DEFS ${CPL_DEFS} UMF_CUDA_ENABLED=1) endif() if(UMF_LEVEL_ZERO_ENABLED) set(INC_DIRS ${INC_DIRS} ${LEVEL_ZERO_INCLUDE_DIRS}) set(LIB_DIRS ${LIB_DIRS} ${ZE_LOADER_LIBRARY_DIRS}) + set(CPL_DEFS ${CPL_DEFS} UMF_LEVEL_ZERO_ENABLED=1) endif() if(UMF_POOL_JEMALLOC_ENABLED) @@ -358,6 +360,10 @@ if(LINUX) # OS-specific functions are implemented NAME provider_tracking_fixture_tests SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) + add_umf_test( + NAME provider_props + SRCS props/provider_props.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) @@ -422,6 +428,12 @@ if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_level_zero_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_ZE_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_props_level_zero + SRCS props/provider_props_level_zero.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} ze_loader) endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) @@ -454,6 +466,11 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_cuda_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_CU_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_props_cuda + SRCS props/provider_props_cuda.cpp providers/cuda_helpers.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} cuda) else() message( STATUS diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 00621b311..3cb8d012b 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -130,6 +130,24 @@ typedef struct provider_base_t { [[maybe_unused]] size_t size) noexcept { return UMF_RESULT_ERROR_UNKNOWN; } + + umf_result_t ext_ctl([[maybe_unused]] umf_ctl_query_source_t source, + [[maybe_unused]] const char *name, + [[maybe_unused]] void *arg, + [[maybe_unused]] size_t size, + [[maybe_unused]] umf_ctl_query_type_t queryType, + [[maybe_unused]] va_list args) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + + umf_result_t ext_get_allocation_properties( + [[maybe_unused]] const void *ptr, + [[maybe_unused]] umf_memory_property_id_t memory_property_id, + [[maybe_unused]] void *value, + [[maybe_unused]] size_t max_property_size) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + virtual ~provider_base_t() = default; } provider_base_t; diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 380cba47d..acb1d124f 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -5,9 +5,10 @@ #include #include -#include "provider_null.h" #include +#include "provider_null.h" + static umf_result_t nullInitialize(const void *params, void **pool) { (void)params; *pool = NULL; @@ -133,6 +134,18 @@ static umf_result_t nullCloseIpcHandle(void *provider, void *ptr, size_t size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +nullGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t propertyId, void *value, + size_t max_property_size) { + (void)provider; + (void)ptr; + (void)propertyId; + (void)value; + (void)max_property_size; + return UMF_RESULT_SUCCESS; +} + umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = nullInitialize, @@ -152,4 +165,5 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .ext_put_ipc_handle = nullPutIpcHandle, .ext_open_ipc_handle = nullOpenIpcHandle, .ext_close_ipc_handle = nullCloseIpcHandle, + .ext_get_allocation_properties = nullGetAllocationProperties, }; diff --git a/test/common/provider_trace.c b/test/common/provider_trace.c index adb808336..e56d2d462 100644 --- a/test/common/provider_trace.c +++ b/test/common/provider_trace.c @@ -5,10 +5,12 @@ #include #include -#include "provider_trace.h" #include #include +#include "memory_props_internal.h" +#include "provider_trace.h" + static umf_result_t traceInitialize(const void *params, void **pool) { umf_provider_trace_params_t *trace_pool = (umf_provider_trace_params_t *)malloc( @@ -214,4 +216,6 @@ umf_memory_provider_ops_t UMF_TRACE_PROVIDER_OPS = { .ext_put_ipc_handle = tracePutIpcHandle, .ext_open_ipc_handle = traceOpenIpcHandle, .ext_close_ipc_handle = traceCloseIpcHandle, + .ext_ctl = NULL, + .ext_get_allocation_properties = NULL, }; diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index a1768b1f6..cddf312f0 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -3,15 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF provider API -#include "provider.hpp" -#include "provider_null.h" -#include "test_helpers.h" - #include #include #include #include +#include "memory_props_internal.h" +#include "provider.hpp" +#include "provider_null.h" +#include "test_helpers.h" + using umf_test::test; TEST_F(test, memoryProviderTrace) { diff --git a/test/props/provider_props.cpp b/test/props/provider_props.cpp new file mode 100644 index 000000000..b8abef9bd --- /dev/null +++ b/test/props/provider_props.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include + +#include "memory_props_internal.h" +#include "provider.hpp" +#include "provider_props.hpp" + +void createFixedProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + constexpr size_t buffer_size = 1024 * 1024; + + void *memory_buffer = malloc(buffer_size); + ASSERT_NE(memory_buffer, nullptr); + + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = + umfFixedMemoryProviderParamsCreate(memory_buffer, buffer_size, ¶ms); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + res = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + out_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(out_provider, nullptr); + + umfFixedMemoryProviderParamsDestroy(params); + + *(uintptr_t *)out_data = (uintptr_t)memory_buffer; +} + +void destroyFixedProvider(umf_memory_provider_handle_t provider, void *data) { + umfMemoryProviderDestroy(provider); + free(data); +} + +void createOsMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; + umf_result_t res = + umfOsMemoryProviderParamsCreate(&os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider_params, nullptr); + + umf_memory_provider_handle_t os_memory_provider = nullptr; + res = + umfMemoryProviderCreate(umfOsMemoryProviderOps(), + os_memory_provider_params, &os_memory_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider, nullptr); + + res = umfOsMemoryProviderParamsDestroy(os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + *out_provider = os_memory_provider; + *(uintptr_t *)out_data = (uintptr_t)NULL; +} + +void destroyOsMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + (void)data; // unused + + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P( + providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createFixedProvider, destroyFixedProvider, + "fixedProvider"}, + testParams{createOsMemoryProvider, + destroyOsMemoryProvider, "osMemoryProvider"}), + nameGen); + +TEST_F(test, CustomPropsTest) { + const uint64_t custom_property_id = UMF_MEMORY_PROPERTY_MAX_RESERVED + 1; + + struct memory_provider : public umf_test::provider_base_t { + umf_result_t alloc(size_t size, size_t alignment, void **ptr) noexcept { + *ptr = umf_ba_global_aligned_alloc(size, alignment); + return UMF_RESULT_SUCCESS; + } + + umf_result_t free(void *ptr, [[maybe_unused]] size_t size) noexcept { + umf_ba_global_free(ptr); + return UMF_RESULT_SUCCESS; + } + + umf_result_t + get_min_page_size([[maybe_unused]] const void *ptr, + [[maybe_unused]] size_t *pageSize) noexcept { + *pageSize = 1024; + return UMF_RESULT_SUCCESS; + } + + umf_result_t ext_get_allocation_properties( + const void *ptr, umf_memory_property_id_t memory_property_id, + void *value, size_t max_property_size) { + + (void)ptr; // unused + + if (memory_property_id == custom_property_id) { + if (max_property_size < sizeof(uint64_t)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(uint64_t *)value = 42; // Custom value for the property + return UMF_RESULT_SUCCESS; + } + + return umf_test::provider_base_t::ext_get_allocation_properties( + ptr, memory_property_id, value, max_property_size); + } + }; + + umf_memory_provider_ops_t provider_ops = + umf_test::providerMakeCOps(); + + umf_memory_provider_handle_t provider = nullptr; + umf_result_t res = + umfMemoryProviderCreate(&provider_ops, nullptr, &provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = nullptr; + res = umfPoolCreate(umfProxyPoolOps(), provider, nullptr, + UMF_POOL_CREATE_FLAG_NONE, &pool); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(pool, nullptr); + + void *ptr = umfPoolMalloc(pool, 1024); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t properties = nullptr; + res = umfGetMemoryPropertiesHandle(ptr, &properties); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(properties, nullptr); + + // get value of the custom property from the properties handle + uint64_t value2 = 0; + res = umfGetMemoryProperty(properties, + (umf_memory_property_id_t)custom_property_id, + sizeof(value2), &value2); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_EQ(value2, 42); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + res = umfPoolDestroy(pool); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + res = umfMemoryProviderDestroy(provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} diff --git a/test/props/provider_props.hpp b/test/props/provider_props.hpp new file mode 100644 index 000000000..5e8c74af9 --- /dev/null +++ b/test/props/provider_props.hpp @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include + +#include "base.hpp" +#include "test_helpers.h" + +using umf_test::test; + +using testParams = + std::tuple, + std::function, + const char *>; + +std::string nameGen(const testing::TestParamInfo param) { + return std::get<2>(param.param); +} + +struct ProviderPropsTest : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + auto [create_fun, destroy_fun, name] = this->GetParam(); + provider_create = create_fun; + provider_destroy = destroy_fun; + (void)name; // unused + + provider_create(&provider, &data); + ASSERT_NE(provider, nullptr); + + umf_result_t umf_result = + umfPoolCreate(umfProxyPoolOps(), provider, nullptr, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + void TearDown() override { + umfPoolDestroy(pool); + provider_destroy(provider, data); + test::TearDown(); + } + + umf_memory_provider_handle_t provider; + umf_memory_pool_handle_t pool; + + std::function provider_create; + std::function provider_destroy; + void *data; +}; + +TEST_P(ProviderPropsTest, genericProps) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_memory_provider_handle_t param_provider = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_PROVIDER_HANDLE, + sizeof(param_provider), ¶m_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_provider, provider); + + umf_memory_pool_handle_t param_pool = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_POOL_HANDLE, + sizeof(param_pool), ¶m_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_pool, pool); + + void *base_address = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(base_address), &base_address); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(base_address, ptr); + + size_t size = 0; + umf_result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, sizeof(size), &size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(size, alloc_size); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(buffer_id), &buffer_id); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, baseAddressFromMiddle) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + void *ptr_mid = (void *)((uintptr_t)ptr + (alloc_size / 2)); + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr_mid, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uintptr_t param_base_address = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(param_base_address), ¶m_base_address); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_base_address, (uintptr_t)ptr); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, uniqueBufferId) { + size_t alloc_size = 8; + size_t num_allocs = 10; + umf_result_t umf_result; + std::set buffer_ids; + + for (size_t i = 0; i < num_allocs; ++i) { + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(buffer_id), &buffer_id); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + // Ensure that the buffer ID is unique by inserting it into a set and + // checking if it was already present + ASSERT_TRUE(buffer_ids.find(buffer_id) == buffer_ids.end()); + ASSERT_TRUE(buffer_ids.insert(buffer_id).second); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +// Negative tests + +TEST_P(ProviderPropsTest, invalidPointer) { + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t umf_result = + umfGetMemoryPropertiesHandle(nullptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); + + uintptr_t invalid_ptr = 0xdeadbeef; + umf_result = + umfGetMemoryPropertiesHandle((void *)invalid_ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); +} + +TEST_P(ProviderPropsTest, invalidPropertyId) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + void *value = nullptr; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_INVALID, + sizeof(value), &value); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertyValue) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(int), NULL); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertySize) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + int value = 0; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + size_t(0), &value); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, nullPropertiesHandle) { + int val = 0; + umf_result_t res = umfGetMemoryProperty( + NULL, UMF_MEMORY_PROPERTY_BASE_ADDRESS, sizeof(val), &val); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/test/props/provider_props_cuda.cpp b/test/props/provider_props_cuda.cpp new file mode 100644 index 000000000..72f6cf17d --- /dev/null +++ b/test/props/provider_props_cuda.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_props.hpp" +#include "providers/cuda_helpers.h" + +void createCudaMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + CUdevice hDevice = -1; + CUcontext hContext = NULL; + + int ret = init_cuda(); + ASSERT_EQ(ret, 0); + + ret = get_cuda_device(&hDevice); + ASSERT_EQ(ret, 0); + ASSERT_NE(hDevice, -1); + + ret = create_context(hDevice, &hContext); + ASSERT_EQ(ret, 0); + ASSERT_NE(hContext, nullptr); + + umf_cuda_memory_provider_params_handle_t cu_params = NULL; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&cu_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(cu_params, nullptr); + + umf_result = umfCUDAMemoryProviderParamsSetContext(cu_params, hContext); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetDevice(cu_params, hDevice); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType( + cu_params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), cu_params, + &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umfCUDAMemoryProviderParamsDestroy(cu_params); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void destroyCudaMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + destroy_context((CUcontext)data); + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createCudaMemoryProvider, + destroyCudaMemoryProvider, + "cudaMemoryProvider"}), + nameGen); diff --git a/test/props/provider_props_level_zero.cpp b/test/props/provider_props_level_zero.cpp new file mode 100644 index 000000000..23f1bbc4e --- /dev/null +++ b/test/props/provider_props_level_zero.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_props.hpp" +#include "utils/utils_level_zero.h" + +void levelZeroMemoryProviderCreate(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + ze_driver_handle_t hDriver = nullptr; + ze_device_handle_t hDevice = nullptr; + ze_context_handle_t hContext = nullptr; + uint32_t driver_idx = 0; + + int ret = utils_ze_init_level_zero(); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_driver_with_gpu(&driver_idx, &hDriver); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_gpu_device(hDriver, &hDevice); + ASSERT_EQ(ret, 0); + + ret = utils_ze_create_context(hDriver, &hContext); + ASSERT_EQ(ret, 0); + + umf_level_zero_memory_provider_params_handle_t params = nullptr; + umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(¶ms); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetContext(params, hContext); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetDevice(params, hDevice); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetMemoryType( + params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + result = umfLevelZeroMemoryProviderParamsDestroy(params); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void levelZeroMemoryProviderDestroy(umf_memory_provider_handle_t provider, + void *data) { + umfMemoryProviderDestroy(provider); + utils_ze_destroy_context((ze_context_handle_t)data); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{ + levelZeroMemoryProviderCreate, + levelZeroMemoryProviderDestroy, + "levelZeroProvider"}), + nameGen); diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 1b448eaf0..ed7e473bc 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include "cuda_helpers.h" @@ -558,6 +559,87 @@ TEST_P(umfCUDAProviderTest, multiContext) { ASSERT_EQ(ret, 0); } +TEST_P(umfCUDAProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + sizeof(baseSize), &baseSize); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(bufferId), &bufferId); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (expected_memory_type != UMF_MEMORY_TYPE_HOST) { + CUdevice device = -1; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + sizeof(device), &device); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, cudaTestHelper.get_test_device()); + } + + CUcontext context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + sizeof(context), &context); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, cudaTestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + struct umfCUDAProviderAllocFlagsTest : umf_test::test, ::testing::WithParamInterface< diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index feb377987..6b977d340 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include "ipcFixtures.hpp" @@ -256,6 +257,7 @@ struct umfLevelZeroProviderTest test::SetUp(); umf_usm_memory_type_t memory_type = this->GetParam(); + umfExpectedMemoryType = memory_type; params = nullptr; memAccessor = nullptr; @@ -308,6 +310,7 @@ struct umfLevelZeroProviderTest std::unique_ptr memAccessor = nullptr; ze_context_handle_t hContext = nullptr; ze_memory_type_t zeMemoryTypeExpected = ZE_MEMORY_TYPE_UNKNOWN; + umf_usm_memory_type_t umfExpectedMemoryType = UMF_MEMORY_TYPE_UNKNOWN; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfLevelZeroProviderTest); @@ -553,6 +556,88 @@ TEST_P(umfLevelZeroProviderTest, setDeviceOrdinalValid) { } } +TEST_P(umfLevelZeroProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + // base address and size + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + sizeof(baseSize), &baseSize); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(bufferId), &bufferId); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (umfExpectedMemoryType != UMF_MEMORY_TYPE_HOST) { + ze_device_handle_t device = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + sizeof(device), &device); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, l0TestHelper.get_test_device()); + } + + ze_context_handle_t context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + sizeof(context), &context); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, l0TestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + // TODO add tests that mixes Level Zero Memory Provider and Disjoint Pool INSTANTIATE_TEST_SUITE_P( diff --git a/test/utils/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp index c9c9d961b..0b758b156 100644 --- a/test/utils/cpp_helpers.hpp +++ b/test/utils/cpp_helpers.hpp @@ -112,6 +112,8 @@ template constexpr umf_memory_provider_ops_t providerOpsBase() { UMF_ASSIGN_OP(ops, T, ext_put_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_open_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_close_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, ext_get_allocation_properties, + UMF_RESULT_ERROR_UNKNOWN); return ops; } } // namespace detail