memory resident limit to enabled peers

Author: lslusarczyk

sycl/cmake/modules/BuildUnifiedRuntime.cmake

@@ -16,6 +16,9 @@ set(UR_BUILD_EXAMPLES "${SYCL_UR_BUILD_TESTS}" CACHE BOOL "" FORCE)
 option(SYCL_UR_FORMAT_CPP_STYLE "Format code style of UR C++ sources" OFF)
 set(UR_FORMAT_CPP_STYLE "${SYCL_UR_FORMAT_CPP_STYLE}" CACHE BOOL "" FORCE)
 
+option(SYCL_UR_ENABLE_ASSERTIONS "Enable assertions for all UR build types" OFF)
+set(UR_ENABLE_ASSERTIONS "${SYCL_UR_ENABLE_ASSERTIONS}" CACHE BOOL "" FORCE)
+
 # Here we override the defaults to unified-runtime
 set(UR_BUILD_XPTI_LIBS OFF CACHE BOOL "")
 set(UR_ENABLE_SYMBOLIZER ON CACHE BOOL "Enable symbolizer for sanitizer layer.")

sycl/source/device.cpp

@@ -219,19 +219,27 @@ bool device::has(aspect Aspect) const { return impl->has(Aspect); }
 void device::ext_oneapi_enable_peer_access(const device &peer) {
  ur_device_handle_t Device = impl->getHandleRef();
  ur_device_handle_t Peer = peer.impl->getHandleRef();
- if (Device != Peer) {
- detail::adapter_impl &Adapter = impl->getAdapter();
- Adapter.call<detail::UrApiKind::urUsmP2PEnablePeerAccessExp>(Device, Peer);
+
+ if (Device == Peer) return;
+
+ if (peer.get_platform() != get_platform()) {
+ throw exception(errc::invalid, "Can not enable peer access between different platforms");
  }
+
+ impl->getAdapter().call<detail::UrApiKind::urUsmP2PEnablePeerAccessExp>(Device, Peer);
 }
 
 void device::ext_oneapi_disable_peer_access(const device &peer) {
  ur_device_handle_t Device = impl->getHandleRef();
  ur_device_handle_t Peer = peer.impl->getHandleRef();
- if (Device != Peer) {
- detail::adapter_impl &Adapter = impl->getAdapter();
- Adapter.call<detail::UrApiKind::urUsmP2PDisablePeerAccessExp>(Device, Peer);
+
+ if (Device == Peer) return;
+
+ if (peer.get_platform() != get_platform()) {
+ throw exception(errc::invalid, "Can not disable peer access between different platforms");
  }
+
+ impl->getAdapter().call<detail::UrApiKind::urUsmP2PDisablePeerAccessExp>(Device, Peer);
 }
 
 bool device::ext_oneapi_can_access_peer(const device &peer,

unified-runtime/cmake/Assertions.cmake

@@ -8,6 +8,7 @@ if(UR_ENABLE_ASSERTIONS)
  # MSVC doesn't like _DEBUG on release builds
  if( NOT MSVC )
  add_compile_definitions(_DEBUG)
+ add_compile_definitions(UR_DASSERT_ENABLED)
  endif()
  # On non-Debug builds cmake automatically defines NDEBUG, so we
  # explicitly undefine it:

unified-runtime/source/adapters/level_zero/CMakeLists.txt

@@ -150,7 +150,6 @@ if(UR_BUILD_ADAPTER_L0_V2)
  ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp
@@ -191,6 +190,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
  ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm_p2p.cpp
  )
  install_ur_library(ur_adapter_level_zero_v2)
 

unified-runtime/source/adapters/level_zero/adapter.cpp

@@ -569,6 +569,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
  if (err == UR_RESULT_SUCCESS) {
  Platforms = std::move(platforms);
  } else {
+ UR_LOG(ERR, "Failed to initialize Platforms");
  throw err;
  }
 }

unified-runtime/source/adapters/level_zero/common.hpp

@@ -218,8 +218,11 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString);
 #define ZE2UR_CALL_THROWS(ZeName, ZeArgs) \
  { \
  ze_result_t ZeResult = ZeName ZeArgs; \
- if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) \
+ if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) { \
+ UR_DFAILURE("failed ZE call " #ZeName " with " #ZeArgs ", with result:" \
+ << Result); \
  throw ze2urResult(Result); \
+ } \
  }
 
 // Perform traced call to L0 without checking for errors

unified-runtime/source/adapters/level_zero/context.cpp

@@ -42,16 +42,18 @@ ur_result_t urContextCreate(
 
  Context->initialize();
  *RetContext = reinterpret_cast<ur_context_handle_t>(Context);
+ // TODO: delete below 'if' when memory isolation in the context is
+ // implemented in the driver
  if (IndirectAccessTrackingEnabled) {
  std::scoped_lock<ur_shared_mutex> Lock(Platform->ContextsMutex);
  Platform->Contexts.push_back(*RetContext);
  }
  } catch (const std::bad_alloc &) {
  return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
- } catch (umf_result_t e) {
- return umf::umf2urResult(e);
- } catch (...) {
- return UR_RESULT_ERROR_UNKNOWN;
+  } catch (umf_result_t e) {
+  return umf::umf2urResult(e);
+  } catch (...) {
+  return UR_RESULT_ERROR_UNKNOWN;
  }
 
  return UR_RESULT_SUCCESS;

unified-runtime/source/adapters/level_zero/device.cpp

@@ -2190,3 +2190,24 @@ void ZeUSMImportExtension::doZeUSMRelease(ze_driver_handle_t DriverHandle,
  void *HostPtr) {
  ZE_CALL_NOCHECK(zexDriverReleaseImportedPointer, (DriverHandle, HostPtr));
 }
+
+std::ostream &operator<<(std::ostream &os,
+ ur_device_handle_t_ const &device_handle) {
+ if (device_handle.Id.has_value()) {
+ return os << device_handle.Id.value();
+ }
+ return os << "NONE";
+}
+
+std::ostream &operator<<(std::ostream &os,
+ ur_device_handle_t_::PeerStatus peer_status) {
+ switch (peer_status) {
+ case ur_device_handle_t_::PeerStatus::DISABLED:
+ return os << "DISABLED";
+ case ur_device_handle_t_::PeerStatus::ENABLED:
+ return os << "ENABLED";
+ case ur_device_handle_t_::PeerStatus::NO_CONNECTION:
+ return os << "NO_CONNECTION";
+ }
+ return os << "UNKNOWN";
+}

unified-runtime/source/adapters/level_zero/device.hpp

@@ -254,17 +254,29 @@ struct ur_device_handle_t_ : ur_object {
  std::unordered_map<ur_exp_image_native_handle_t, ze_image_handle_t>
  ZeOffsetToImageHandleMap;
 
+ // Devices which user enabled p2p access by
+ // urUsmP2P(Enable|Disable)PeerAccessExp. Devices are indexed by device id.
+ enum class PeerStatus : char { ENABLED, DISABLED, NO_CONNECTION };
+ std::vector<PeerStatus>
+ peers; // info if our device can access given peer device allocations
+
  // unique ephemeral identifer of the device in the adapter
  std::optional<DeviceId> Id;
 
  ur::RefCount RefCount;
 };
 
+std::ostream &operator<<(std::ostream &os,
+ ur_device_handle_t_ const &device_handle);
+std::ostream &operator<<(std::ostream &os,
+ ur_device_handle_t_::PeerStatus peer_status);
+
 // Collects a flat vector of unique devices for USM memory pool creation.
 // Traverses the input devices and their sub-devices, ensuring each Level Zero
 // device handle appears only once in the result.
 inline std::vector<ur_device_handle_t> CollectDevicesForUsmPoolCreation(
  const std::vector<ur_device_handle_t> &Devices) {
+
  std::vector<ur_device_handle_t> DevicesAndSubDevices;
  std::unordered_set<ze_device_handle_t> Seen;
 

unified-runtime/source/adapters/level_zero/platform.cpp

@@ -630,9 +630,9 @@ ur_platform_handle_t_::getDeviceFromNativeHandle(ze_device_handle_t ZeDevice) {
  std::shared_lock<ur_shared_mutex> Lock(URDevicesCacheMutex);
  auto it = std::find_if(URDevicesCache.begin(), URDevicesCache.end(),
  [&](std::unique_ptr<ur_device_handle_t_> &D) {
- return D.get()->ZeDevice == ZeDevice &&
- (D.get()->RootDevice == nullptr ||
- D.get()->RootDevice->RootDevice == nullptr);
+ return D->ZeDevice == ZeDevice &&
+ (D->RootDevice == nullptr ||
+ D->RootDevice->RootDevice == nullptr);
  });
  if (it != URDevicesCache.end()) {
  return (*it).get();
@@ -785,6 +785,44 @@ ur_result_t ur_platform_handle_t_::populateDeviceCacheIfNeeded() {
  dev->Id = id++;
  }
 
+ for (auto &dev : URDevicesCache) {
+ dev->peers = std::vector<ur_device_handle_t_::PeerStatus>(
+ URDevicesCache.size(), ur_device_handle_t_::PeerStatus::NO_CONNECTION);
+
+ for (size_t peerId = 0; peerId < URDevicesCache.size(); ++peerId) {
+ if (peerId == dev->Id.value())
+ continue;
+
+ ZeStruct<ze_device_p2p_properties_t> p2pProperties;
+ ZE2UR_CALL_THROWS(
+ zeDeviceGetP2PProperties,
+ (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2pProperties));
+ if (!(p2pProperties.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ACCESS)) {
+ UR_LOG(INFO,
+ "p2p access to memory of dev:{} from dev:{} not possible due to "
+ "lack of p2p property",
+ peerId, dev->Id.value());
+ continue;
+ }
+
+ ze_bool_t p2p;
+ ZE2UR_CALL_THROWS(
+ zeDeviceCanAccessPeer,
+ (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2p));
+ if (!p2p) {
+ UR_LOG(INFO,
+ "p2p access to memory of dev:{} from dev:{} not possible due to "
+ "no connection",
+ peerId, dev->Id.value());
+ continue;
+ }
+
+ UR_LOG(INFO, "p2p access to memory of dev:{} from dev:{} can be enabled",
+ peerId, dev->Id.value());
+ dev->peers[peerId] = ur_device_handle_t_::PeerStatus::DISABLED;
+ }
+ }
+
  return UR_RESULT_SUCCESS;
 }