[OFFLOAD] Add plugin with support for Intel oneAPI Level Zero

offload/CMakeLists.txt

@@ -152,7 +152,7 @@ endif()
 
 set(LIBOMPTARGET_ALL_PLUGIN_TARGETS amdgpu cuda host)
 set(LIBOMPTARGET_PLUGINS_TO_BUILD "all" CACHE STRING
- "Semicolon-separated list of plugins to use: cuda, amdgpu, host or \"all\".")
+ "Semicolon-separated list of plugins to use: cuda, amdgpu, level_zero, host or \"all\".")
 
 if(LIBOMPTARGET_PLUGINS_TO_BUILD STREQUAL "all")
  set(LIBOMPTARGET_PLUGINS_TO_BUILD ${LIBOMPTARGET_ALL_PLUGIN_TARGETS})
@@ -176,6 +176,18 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$"
  list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "cuda")
  endif()
 endif()
+if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64)$" AND
+ CMAKE_SYSTEM_NAME MATCHES "Linux|Windows"))
+ if("level_zero" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
+ message(STATUS "Not building Level Zero plugin: it is only supported on "
+ "Linux/Windows x86_64 or ppc64le hosts")
+ list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "level_zero")
+ endif()
+endif()
+if("level_zero" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD AND
+ NOT LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND)
+ list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "level_zero")
+endif()
 message(STATUS "Building the offload library with support for "
  "the \"${LIBOMPTARGET_PLUGINS_TO_BUILD}\" plugins")
 
@@ -214,6 +226,7 @@ set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} riscv64-unknown-linux
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} riscv64-unknown-linux-gnu-LTO")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} loongarch64-unknown-linux-gnu")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} loongarch64-unknown-linux-gnu-LTO")
+set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} spirv64-intel")
 
 # Once the plugins for the different targets are validated, they will be added to
 # the list of supported targets in the current system.

offload/cmake/Modules/LibomptargetGetDependencies.cmake

@@ -42,6 +42,16 @@ endif()
 find_package(FFI QUIET)
 set(LIBOMPTARGET_DEP_LIBFFI_FOUND ${FFI_FOUND})
 
+################################################################################
+# Looking for offload-arch...
+################################################################################
+if(TARGET offload-arch)
+ get_property(LIBOMPTARGET_OFFLOAD_ARCH TARGET offload-arch PROPERTY LOCATION)
+else()
+ find_program(LIBOMPTARGET_OFFLOAD_ARCH NAMES offload-arch
+ PATHS ${LLVM_TOOLS_BINARY_DIR})
+endif()
+
 ################################################################################
 # Looking for NVIDIA GPUs...
 ################################################################################
@@ -89,4 +99,27 @@ if(LIBOMPTARGET_AMDGPU_ARCH)
  endif()
 endif()
 
+################################################################################
+# Looking for Level0
+################################################################################
+find_path(LIBOMPTARGET_DEP_LEVEL_ZERO_INCLUDE_DIR NAMES level_zero/ze_api.h)
+
+if(NOT LIBOMPTARGET_DEP_LEVEL_ZERO_INCLUDE_DIR)
+ set(LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND FALSE)
+else()
+ set(LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND TRUE)
+ find_library(LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY NAMES ze_loader)
+endif()
+
+if(LIBOMPTARGET_OFFLOAD_ARCH)
+ execute_process(COMMAND ${LIBOMPTARGET_OFFLOAD_ARCH} "--only=intel"
+ OUTPUT_VARIABLE LIBOMPTARGET_INTELGPU_ARCH_OUTPUT
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+ string(REPLACE "\n" ";" intelgpu_arch_list "${LIBOMPTARGET_INTELGPU_ARCH_OUTPUT}")
+ if(intelgpu_arch_list)
+ set(LIBOMPTARGET_FOUND_INTELGPU_GPU TRUE)
+ set(LIBOMPTARGET_INTELGPU_DETECTED_ARCH_LIST "${intelgpu_arch_list}")
+ endif()
+endif()
+
 set(OPENMP_PTHREAD_LIB ${LLVM_PTHREAD_LIB})

offload/include/OpenMP/InteropAPI.h

@@ -160,17 +160,12 @@ struct InteropTableEntry {
  Interops.push_back(obj);
  }
 
- template <class ClearFuncTy> void clear(ClearFuncTy f) {
- for (auto &Obj : Interops) {
- f(Obj);
- }
- }
-
  /// vector interface
  int size() const { return Interops.size(); }
  iterator begin() { return Interops.begin(); }
  iterator end() { return Interops.end(); }
  iterator erase(iterator it) { return Interops.erase(it); }
+ void clear() { Interops.clear(); }
 };
 
 struct InteropTblTy

offload/include/PerThreadTable.h

@@ -14,19 +14,96 @@
 #define OFFLOAD_PERTHREADTABLE_H
 
 #include <list>
+#include <llvm/Support/Error.h>
 #include <memory>
 #include <mutex>
+#include <type_traits>
+
+template <typename ObjectType> struct PerThread {
+ struct PerThreadData {
+ std::unique_ptr<ObjectType> ThEntry;
+ };
+
+ std::mutex Mtx;
+ std::list<std::shared_ptr<PerThreadData>> ThreadDataList;
+
+ // define default constructors, disable copy and move constructors
+ PerThread() = default;
+ PerThread(const PerThread &) = delete;
+ PerThread(PerThread &&) = delete;
+ PerThread &operator=(const PerThread &) = delete;
+ PerThread &operator=(PerThread &&) = delete;
+ ~PerThread() {
+ std::lock_guard<std::mutex> Lock(Mtx);
+ ThreadDataList.clear();
+ }
+
+private:
+ PerThreadData &getThreadData() {
+ static thread_local std::shared_ptr<PerThreadData> ThreadData = nullptr;
+ if (!ThreadData) {
+ ThreadData = std::make_shared<PerThreadData>();
+ std::lock_guard<std::mutex> Lock(Mtx);
+ ThreadDataList.push_back(ThreadData);
+ }
+ return *ThreadData;
+ }
+
+protected:
+ ObjectType &getThreadEntry() {
+ auto &ThData = getThreadData();
+ if (ThData.ThEntry)
+ return *ThData.ThEntry;
+ ThData.ThEntry = std::make_unique<ObjectType>();
+ return *ThData.ThEntry;
+ }
+
+public:
+ ObjectType &get() { return getThreadEntry(); }
+
+ template <class F> void clear(F f) {
+ std::lock_guard<std::mutex> Lock(Mtx);
+ for (auto ThData : ThreadDataList) {
+ if (!ThData->ThEntry)
+ continue;
+ f(*ThData->ThEntry);
+ }
+ ThreadDataList.clear();
+ }
+};
 
 // Using an STL container (such as std::vector) indexed by thread ID has
 // too many race conditions issues so we store each thread entry into a
 // thread_local variable.
 // T is the container type used to store the objects, e.g., std::vector,
 // std::set, etc. by each thread. O is the type of the stored objects e.g.,
 // omp_interop_val_t *, ...
-
 template <typename ContainerType, typename ObjectType> struct PerThreadTable {
  using iterator = typename ContainerType::iterator;
 
+ template <typename, typename = std::void_t<>>
+ struct has_iterator : std::false_type {};
+ template <typename T>
+ struct has_iterator<T, std::void_t<typename T::iterator>> : std::true_type {};
+
+ template <typename T, typename = std::void_t<>>
+ struct has_clear : std::false_type {};
+ template <typename T>
+ struct has_clear<T, std::void_t<decltype(std::declval<T>().clear())>>
+ : std::true_type {};
+
+ template <typename T, typename = std::void_t<>>
+ struct has_clearAll : std::false_type {};
+ template <typename T>
+ struct has_clearAll<T, std::void_t<decltype(std::declval<T>().clearAll(1))>>
+ : std::true_type {};
+
+ template <typename, typename = std::void_t<>>
+ struct is_associative : std::false_type {};
+ template <typename T>
+ struct is_associative<T, std::void_t<typename T::mapped_type>>
+ : std::true_type {};
+
  struct PerThreadData {
  size_t NElements = 0;
  std::unique_ptr<ContainerType> ThEntry;
@@ -71,6 +148,11 @@ template <typename ContainerType, typename ObjectType> struct PerThreadTable {
  return ThData.NElements;
  }
 
+ void setNElements(size_t Size) {
+ auto &NElements = getThreadNElements();
+ NElements = Size;
+ }
+
 public:
  void add(ObjectType obj) {
  auto &Entry = getThreadEntry();
@@ -104,11 +186,99 @@ template <typename ContainerType, typename ObjectType> struct PerThreadTable {
  for (auto ThData : ThreadDataList) {
  if (!ThData->ThEntry || ThData->NElements == 0)
  continue;
- ThData->ThEntry->clear(f);
+ if constexpr (has_clearAll<ContainerType>::value) {
+ ThData->ThEntry->clearAll(f);
+ } else if constexpr (has_iterator<ContainerType>::value &&
+ has_clear<ContainerType>::value) {
+ for (auto &Obj : *ThData->ThEntry) {
+ if constexpr (is_associative<ContainerType>::value) {
+ f(Obj.second);
+ } else {
+ f(Obj);
+ }
+ }
+ ThData->ThEntry->clear();
+ } else {
+ static_assert(true, "Container type not supported");
+ }
  ThData->NElements = 0;
  }
  ThreadDataList.clear();
  }
+
+ template <class F> llvm::Error deinit(F f) {
+ std::lock_guard<std::mutex> Lock(Mtx);
+ for (auto ThData : ThreadDataList) {
+ if (!ThData->ThEntry || ThData->NElements == 0)
+ continue;
+ for (auto &Obj : *ThData->ThEntry) {
+ if constexpr (is_associative<ContainerType>::value) {
+ if (auto Err = f(Obj.second))
+ return Err;
+ } else {
+ if (auto Err = f(Obj))
+ return Err;
+ }
+ }
+ }
+ return llvm::Error::success();
+ }
+};
+
+template <typename T, typename = std::void_t<>> struct ContainerValueType {
+ using type = typename T::value_type;
+};
+template <typename T>
+struct ContainerValueType<T, std::void_t<typename T::mapped_type>> {
+ using type = typename T::mapped_type;
+};
+
+template <typename ContainerType, size_t reserveSize = 0>
+struct PerThreadContainer
+ : public PerThreadTable<ContainerType,
+ typename ContainerValueType<ContainerType>::type> {
+
+ // helpers
+ template <typename T, typename = std::void_t<>> struct indexType {
+ using type = typename T::size_type;
+ };
+ template <typename T> struct indexType<T, std::void_t<typename T::key_type>> {
+ using type = typename T::key_type;
+ };
+ template <typename T, typename = std::void_t<>>
+ struct has_resize : std::false_type {};
+ template <typename T>
+ struct has_resize<T, std::void_t<decltype(std::declval<T>().resize(1))>>
+ : std::true_type {};
+
+ template <typename T, typename = std::void_t<>>
+ struct has_reserve : std::false_type {};
+ template <typename T>
+ struct has_reserve<T, std::void_t<decltype(std::declval<T>().reserve(1))>>
+ : std::true_type {};
+
+ using IndexType = typename indexType<ContainerType>::type;
+ using ObjectType = typename ContainerValueType<ContainerType>::type;
+
+ // Get the object for the given index in the current thread
+ ObjectType &get(IndexType Index) {
+ auto &Entry = this->getThreadEntry();
+
+ // specialized code for vector-like containers
+ if constexpr (has_resize<ContainerType>::value) {
+ if (Index >= Entry.size()) {
+ if constexpr (has_reserve<ContainerType>::value && reserveSize > 0) {
+ if (Entry.capacity() < reserveSize)
+ Entry.reserve(reserveSize);
+ }
+ // If the index is out of bounds, try resize the container
+ Entry.resize(Index + 1);
+ }
+ }
+ ObjectType &Ret = Entry[Index];
+ this->setNElements(Entry.size());
+ return Ret;
+ }
 };
 
 #endif

offload/include/omptarget.h

@@ -101,6 +101,7 @@ enum TargetAllocTy : int32_t {
  TARGET_ALLOC_HOST,
  TARGET_ALLOC_SHARED,
  TARGET_ALLOC_DEFAULT,
+ TARGET_ALLOC_LAST = TARGET_ALLOC_DEFAULT
 };
 
 inline KernelArgsTy CTorDTorKernelArgs = {

offload/liboffload/API/Platform.td

@@ -27,6 +27,7 @@ def ol_platform_backend_t : Enum {
  Etor<"UNKNOWN", "The backend is not recognized">,
  Etor<"CUDA", "The backend is CUDA">,
  Etor<"AMDGPU", "The backend is AMDGPU">,
+ Etor<"LEVEL_ZERO", "The backend is Level Zero">,
  Etor<"HOST", "The backend is the host">,
  ];
 }

offload/liboffload/src/OffloadImpl.cpp

@@ -263,6 +263,8 @@ constexpr ol_platform_backend_t pluginNameToBackend(StringRef Name) {
  return OL_PLATFORM_BACKEND_AMDGPU;
  } else if (Name == "cuda") {
  return OL_PLATFORM_BACKEND_CUDA;
+ } else if (Name == "level_zero") {
+ return OL_PLATFORM_BACKEND_LEVEL_ZERO;
  } else {
  return OL_PLATFORM_BACKEND_UNKNOWN;
  }

offload/plugins-nextgen/common/include/DLWrap.h

@@ -282,5 +282,21 @@ template <size_t Requested, size_t Required> constexpr void verboseAssert() {
  return dlwrap::SYM_USE##_Trait::get()(x0, x1, x2, x3, x4, x5, x6, x7, x8, \
  x9, x10); \
  }
+#define DLWRAP_INSTANTIATE_12(SYM_DEF, SYM_USE, T) \
+ T::ReturnType SYM_DEF(typename T::template arg<0>::type x0, \
+ typename T::template arg<1>::type x1, \
+ typename T::template arg<2>::type x2, \
+ typename T::template arg<3>::type x3, \
+ typename T::template arg<4>::type x4, \
+ typename T::template arg<5>::type x5, \
+ typename T::template arg<6>::type x6, \
+ typename T::template arg<7>::type x7, \
+ typename T::template arg<8>::type x8, \
+ typename T::template arg<9>::type x9, \
+ typename T::template arg<10>::type x10, \
+ typename T::template arg<11>::type x11) { \
+ return dlwrap::SYM_USE##_Trait::get()(x0, x1, x2, x3, x4, x5, x6, x7, x8, \
+ x9, x10, x11); \
+ }
 
 #endif // OMPTARGET_SHARED_DLWRAP_H

offload/plugins-nextgen/common/include/PluginInterface.h

@@ -1266,7 +1266,7 @@ struct GenericPluginTy {
  virtual GenericGlobalHandlerTy *createGlobalHandler() = 0;
 
  /// Get the reference to the device with a certain device id.
- GenericDeviceTy &getDevice(int32_t DeviceId) {
+ GenericDeviceTy &getDevice(int32_t DeviceId) const {
  assert(isValidDeviceId(DeviceId) && "Invalid device id");
  assert(Devices[DeviceId] && "Device is uninitialized");
 
@@ -1527,6 +1527,23 @@ struct GenericPluginTy {
  /// object and return immediately.
  int32_t async_barrier(omp_interop_val_t *Interop);
 
+ struct DevicesRangeTy {
+ using iterator = llvm::SmallVector<GenericDeviceTy *>::iterator;
+
+ iterator BeginIt;
+ iterator EndIt;
+
+ DevicesRangeTy(iterator BeginIt, iterator EndIt)
+ : BeginIt(BeginIt), EndIt(EndIt) {}
+
+ auto &begin() { return BeginIt; }
+ auto &end() { return EndIt; }
+ };
+
+ DevicesRangeTy getDevicesRange() {
+ return DevicesRangeTy(Devices.begin(), Devices.end());
+ }
+
 private:
  /// Indicates if the platform runtime has been fully initialized.
  bool Initialized = false;