platform_impl.cpp

//==----------- platform_impl.cpp ------------------------------------------==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include <detail/allowlist.hpp>
#include <detail/config.hpp>
#include <detail/device_impl.hpp>
#include <detail/global_handler.hpp>
#include <detail/platform_impl.hpp>
#include <detail/ur_info_code.hpp>
#include <sycl/backend_types.hpp>
#include <sycl/detail/iostream_proxy.hpp>
#include <sycl/detail/ur.hpp>
#include <sycl/detail/util.hpp>
#include <sycl/device.hpp>
#include <sycl/info/info_desc.hpp>

#include <algorithm>
#include <cstring>
#include <mutex>
#include <string>
#include <unordered_set>
#include <vector>

namespace sycl {
inline namespace _V1 {
namespace detail {

platform_impl::platform_impl(ur_platform_handle_t APlatform,
                             adapter_impl &Adapter)
    : MPlatform(APlatform), MAdapter(&Adapter) {

  // Find out backend of the platform
  ur_backend_t UrBackend = UR_BACKEND_UNKNOWN;
  Adapter.call_nocheck<UrApiKind::urPlatformGetInfo>(
      APlatform, UR_PLATFORM_INFO_BACKEND, sizeof(ur_backend_t), &UrBackend,
      nullptr);
  MBackend = convertUrBackend(UrBackend);
}

platform_impl::~platform_impl() = default;

platform_impl &
platform_impl::getOrMakePlatformImpl(ur_platform_handle_t UrPlatform,
                                     adapter_impl &Adapter) {
  std::shared_ptr<platform_impl> Result;
  {
    const std::lock_guard<std::mutex> Guard(
        GlobalHandler::instance().getPlatformMapMutex());

    std::vector<std::shared_ptr<platform_impl>> &PlatformCache =
        GlobalHandler::instance().getPlatformCache();

    // If we've already seen this platform, return the impl
    for (const auto &PlatImpl : PlatformCache) {
      if (PlatImpl->getHandleRef() == UrPlatform)
        return *PlatImpl;
    }

    // Otherwise make the impl. Our ctor/dtor are private, so std::make_shared
    // needs a bit of help...
    struct creator : platform_impl {
      creator(ur_platform_handle_t APlatform, adapter_impl &AAdapter)
          : platform_impl(APlatform, AAdapter) {}
    };
    Result = std::make_shared<creator>(UrPlatform, Adapter);
    PlatformCache.emplace_back(Result);
  }

  return *Result;
}

platform_impl &
platform_impl::getPlatformFromUrDevice(ur_device_handle_t UrDevice,
                                       adapter_impl &Adapter) {
  ur_platform_handle_t Plt =
      nullptr; // TODO catch an exception and put it to list
  // of asynchronous exceptions
  Adapter.call<UrApiKind::urDeviceGetInfo>(UrDevice, UR_DEVICE_INFO_PLATFORM,
                                           sizeof(Plt), &Plt, nullptr);
  return getOrMakePlatformImpl(Plt, Adapter);
}

context_impl &platform_impl::khr_get_default_context() {
  GlobalHandler &GH = GlobalHandler::instance();
  // Keeping the default context for platforms in the global cache to avoid
  // shared_ptr based circular dependency between platform and context classes
  std::unordered_map<platform_impl *, std::shared_ptr<context_impl>>
      &PlatformToDefaultContextCache = GH.getPlatformToDefaultContextCache();

  std::lock_guard<std::mutex> Lock{GH.getPlatformToDefaultContextCacheMutex()};

  auto It = PlatformToDefaultContextCache.find(this);
  if (PlatformToDefaultContextCache.end() == It)
    std::tie(It, std::ignore) = PlatformToDefaultContextCache.insert(
        {this, detail::getSyclObjImpl(context{get_devices()})});

  return *It->second;
}

// Get the vector of platforms supported by a given UR adapter
// replace uses of this with a helper in adapter object, the adapter
// objects will own the ur adapter handles and they'll need to pass them to
// urPlatformsGet - so urPlatformsGet will need to be wrapped with a helper
std::vector<platform> platform_impl::getAdapterPlatforms(adapter_impl &Adapter,
                                                         bool Supported) {
  std::vector<platform> Platforms;

  auto UrPlatforms = Adapter.getUrPlatforms();

  if (UrPlatforms.empty()) {
    return Platforms;
  }

  for (const auto &UrPlatform : UrPlatforms) {
    platform Platform = detail::createSyclObjFromImpl<platform>(
        getOrMakePlatformImpl(UrPlatform, Adapter));
    bool HasAnyDevices = !Platform.get_devices(info::device_type::all).empty();

    if (!Supported) {
      if (!HasAnyDevices) {
        Platforms.push_back(std::move(Platform));
      }
    } else {
      // The SYCL spec says that a platform has one or more devices. ( SYCL
      // 2020 4.6.2 ) If we have an empty platform, we don't report it back
      // from platform::get_platforms().
      if (HasAnyDevices) {
        Platforms.push_back(std::move(Platform));
      }
    }
  }
  return Platforms;
}

// This routine has the side effect of registering each platform's last device
// id into each adapter, which is used for device counting.
std::vector<platform> platform_impl::get_platforms() {

  // See which platform we want to be served by which adapter.
  // There should be just one adapter serving each backend.
  std::vector<adapter_impl *> &Adapters = ur::initializeUr();
  std::vector<std::pair<platform, adapter_impl *>> PlatformsWithAdapter;

  // Then check backend-specific adapters
  for (auto &Adapter : Adapters) {
    const auto &AdapterPlatforms = getAdapterPlatforms(*Adapter);
    for (const auto &P : AdapterPlatforms) {
      PlatformsWithAdapter.push_back({P, Adapter});
    }
  }

  // For the selected platforms register them with their adapters
  std::vector<platform> Platforms;
  for (auto &Platform : PlatformsWithAdapter) {
    auto &Adapter = Platform.second;
    std::lock_guard<std::mutex> Guard(*Adapter->getAdapterMutex());
    Adapter->getPlatformId(getSyclObjImpl(Platform.first)->getHandleRef());
    Platforms.push_back(Platform.first);
  }

  // This initializes a function-local variable whose destructor is invoked as
  // the SYCL shared library is first being unloaded.
  GlobalHandler::registerStaticVarShutdownHandler();

  return Platforms;
}

// Since ONEAPI_DEVICE_SELECTOR admits negative filters, we use type traits
// to distinguish the case where we are working with ONEAPI_DEVICE_SELECTOR
// in the places where the functionality diverges between these two
// environment variables.
// The return value is a vector that represents the indices of the chosen
// devices.
template <typename ListT, typename FilterT>
std::vector<int>
platform_impl::filterDeviceFilter(std::vector<ur_device_handle_t> &UrDevices,
                                  ListT *FilterList) const {

  constexpr bool is_ods_target = std::is_same_v<FilterT, ods_target>;

  if constexpr (is_ods_target) {

    // Since we are working with ods_target filters ,which can be negative,
    // we sort the filters so that all the negative filters appear before
    // all the positive filters.  This enables us to have the full list of
    // blacklisted devices by the time we get to the positive filters
    // so that if a positive filter matches a blacklisted device we do
    // not add it to the list of available devices.
    std::sort(FilterList->get().begin(), FilterList->get().end(),
              [](const ods_target &filter1, const ods_target &filter2) {
                return filter1.IsNegativeTarget && !filter2.IsNegativeTarget;
              });
  }

  // this map keeps track of devices discarded by negative filters, it is only
  // used in the ONEAPI_DEVICE_SELECTOR implemenation. It cannot be placed
  // in the if statement above because it will then be out of scope in the rest
  // of the function
  std::map<int, bool> Blacklist;
  // original indices keeps track of the device numbers of the chosen
  // devices and is whats returned by the function
  std::vector<int> original_indices;

  // Find out backend of the platform
  ur_backend_t UrBackend = UR_BACKEND_UNKNOWN;
  MAdapter->call<UrApiKind::urPlatformGetInfo>(
      MPlatform, UR_PLATFORM_INFO_BACKEND, sizeof(ur_backend_t), &UrBackend,
      nullptr);
  backend Backend = convertUrBackend(UrBackend);

  int InsertIDx = 0;
  // DeviceIds should be given consecutive numbers across platforms in the same
  // backend
  std::lock_guard<std::mutex> Guard(*MAdapter->getAdapterMutex());
  int DeviceNum = MAdapter->getStartingDeviceId(MPlatform);
  for (ur_device_handle_t Device : UrDevices) {
    ur_device_type_t UrDevType = UR_DEVICE_TYPE_ALL;
    MAdapter->call<UrApiKind::urDeviceGetInfo>(Device, UR_DEVICE_INFO_TYPE,
                                               sizeof(ur_device_type_t),
                                               &UrDevType, nullptr);
    info::device_type DeviceType = detail::ConvertDeviceType(UrDevType);

    for (const FilterT &Filter : FilterList->get()) {
      backend FilterBackend = Filter.Backend.value_or(backend::all);
      // First, match the backend entry.
      if (FilterBackend != Backend && FilterBackend != backend::all)
        continue;
      info::device_type FilterDevType =
          Filter.DeviceType.value_or(info::device_type::all);

      // Match the device_num entry.
      if (Filter.DeviceNum && DeviceNum != Filter.DeviceNum.value())
        continue;

      if (FilterDevType != info::device_type::all &&
          FilterDevType != DeviceType)
        continue;

      if constexpr (is_ods_target) {
        // Dealing with ONEAPI_DEVICE_SELECTOR - check for negative filters.
        if (Blacklist[DeviceNum]) // already blacklisted.
          break;

        if (Filter.IsNegativeTarget) {
          // Filter is negative and the device matches the filter so
          // blacklist the device now.
          Blacklist[DeviceNum] = true;
          break;
        }
      }

      UrDevices[InsertIDx++] = Device;
      original_indices.push_back(DeviceNum);
      break;
    }
    DeviceNum++;
  }
  UrDevices.resize(InsertIDx);
  // remember the last backend that has gone through this filter function
  // to assign a unique device id number across platforms that belong to
  // the same backend. For example, opencl:cpu:0, opencl:acc:1, opencl:gpu:2
  MAdapter->setLastDeviceId(MPlatform, DeviceNum);
  return original_indices;
}

device_impl *platform_impl::getDeviceImpl(ur_device_handle_t UrDevice) {
  const std::lock_guard<std::mutex> Guard(MDeviceMapMutex);
  return getDeviceImplHelper(UrDevice);
}

device_impl &platform_impl::getOrMakeDeviceImpl(ur_device_handle_t UrDevice) {
  const std::lock_guard<std::mutex> Guard(MDeviceMapMutex);
  // If we've already seen this device, return the impl
  if (device_impl *Result = getDeviceImplHelper(UrDevice))
    return *Result;

  // Otherwise make the impl
  MDevices.emplace_back(std::make_unique<device_impl>(
      UrDevice, *this, device_impl::private_tag{}));

  return *MDevices.back();
}

static bool supportsAffinityDomain(const device &dev,
                                   info::partition_property partitionProp,
                                   info::partition_affinity_domain domain) {
  if (partitionProp != info::partition_property::partition_by_affinity_domain) {
    return true;
  }
  auto supported = dev.get_info<info::device::partition_affinity_domains>();
  auto It = std::find(std::begin(supported), std::end(supported), domain);
  return It != std::end(supported);
}

static bool supportsPartitionProperty(const device &dev,
                                      info::partition_property partitionProp) {
  auto supported = dev.get_info<info::device::partition_properties>();
  auto It =
      std::find(std::begin(supported), std::end(supported), partitionProp);
  return It != std::end(supported);
}

static std::vector<device> amendDeviceAndSubDevices(
    backend PlatformBackend, std::vector<device> &DeviceList,
    ods_target_list *OdsTargetList, const std::vector<int> &original_indices,
    platform_impl &PlatformImpl) {
  constexpr info::partition_property partitionProperty =
      info::partition_property::partition_by_affinity_domain;
  constexpr info::partition_affinity_domain affinityDomain =
      info::partition_affinity_domain::next_partitionable;

  std::vector<device> FinalResult;
  // (Only) when amending sub-devices for ONEAPI_DEVICE_SELECTOR, all
  // sub-devices are treated as root.
  TempAssignGuard<bool> TAG(PlatformImpl.MAlwaysRootDevice, true);

  for (unsigned i = 0; i < DeviceList.size(); i++) {
    // device has already been screened. The question is whether it should be a
    // top level device and/or is expected to add its sub-devices to the list.
    device &dev = DeviceList[i];
    bool deviceAdded = false;
    for (ods_target target : OdsTargetList->get()) {
      backend TargetBackend = target.Backend.value_or(backend::all);
      if (PlatformBackend != TargetBackend && TargetBackend != backend::all)
        continue;

      bool deviceMatch = target.HasDeviceWildCard; // opencl:*
      if (target.DeviceType) {                     // opencl:gpu
        deviceMatch =
            ((target.DeviceType == info::device_type::all) ||
             (dev.get_info<info::device::device_type>() == target.DeviceType));

      } else if (target.DeviceNum) { // opencl:0
        deviceMatch = (target.DeviceNum.value() == original_indices[i]);
      }

      if (!deviceMatch)
        continue;

      // Top level matches. Do we add it, or subdevices, or sub-sub-devices?
      bool wantSubDevice = target.SubDeviceNum || target.HasSubDeviceWildCard;
      bool supportsSubPartitioning =
          (supportsPartitionProperty(dev, partitionProperty) &&
           supportsAffinityDomain(dev, partitionProperty, affinityDomain));
      bool wantSubSubDevice =
          target.SubSubDeviceNum || target.HasSubSubDeviceWildCard;

      if (!wantSubDevice) {
        // -- Add top level device only.
        if (!deviceAdded) {
          FinalResult.push_back(dev);
          deviceAdded = true;
        }
        continue;
      }

      if (!supportsSubPartitioning) {
        if (target.DeviceNum ||
            (target.DeviceType &&
             (target.DeviceType.value() != info::device_type::all))) {
          // This device was specifically requested and yet is not
          // partitionable.
          std::cout << "device is not partitionable: " << target << std::endl;
        }
        continue;
      }

      auto subDevices = dev.create_sub_devices<
          info::partition_property::partition_by_affinity_domain>(
          affinityDomain);
      if (target.SubDeviceNum) {
        if (subDevices.size() <= target.SubDeviceNum.value()) {
          std::cout << "subdevice index out of bounds: " << target << std::endl;
          continue;
        }
        subDevices[0] = subDevices[target.SubDeviceNum.value()];
        subDevices.resize(1);
      }

      if (!wantSubSubDevice) {
        // -- Add sub device(s) only.
        FinalResult.insert(FinalResult.end(), subDevices.begin(),
                           subDevices.end());
        continue;
      }

      // -- Add sub sub device(s).
      for (device subDev : subDevices) {
        bool supportsSubSubPartitioning =
            (supportsPartitionProperty(subDev, partitionProperty) &&
             supportsAffinityDomain(subDev, partitionProperty, affinityDomain));
        if (!supportsSubSubPartitioning) {
          if (target.SubDeviceNum) {
            // Parent subdevice was specifically requested, yet is not
            // partitionable.
            std::cout << "sub-device is not partitionable: " << target
                      << std::endl;
          }
          continue;
        }

        // Allright, lets get them sub-sub-devices.
        auto subSubDevices =
            subDev.create_sub_devices<partitionProperty>(affinityDomain);
        if (target.SubSubDeviceNum) {
          if (subSubDevices.size() <= target.SubSubDeviceNum.value()) {
            std::cout << "sub-sub-device index out of bounds: " << target
                      << std::endl;
            continue;
          }
          subSubDevices[0] = subSubDevices[target.SubSubDeviceNum.value()];
          subSubDevices.resize(1);
        }
        FinalResult.insert(FinalResult.end(), subSubDevices.begin(),
                           subSubDevices.end());
      }
    }
  }
  return FinalResult;
}

std::vector<device>
platform_impl::get_devices(info::device_type DeviceType) const {
  std::vector<device> Res;
  // Host is no longer supported, so it returns an empty vector.
  if (DeviceType == info::device_type::host)
    return std::vector<device>{};

  // For custom devices, UR has additional type enums.
  if (DeviceType == info::device_type::custom) {
    getDevicesImplHelper(UR_DEVICE_TYPE_CUSTOM, Res);
    getDevicesImplHelper(UR_DEVICE_TYPE_MCA, Res);
    getDevicesImplHelper(UR_DEVICE_TYPE_VPU, Res);
    // Some backends may return the MCA and VPU types as part of custom, so
    // remove duplicates.
    std::sort(Res.begin(), Res.end(),
              [](const sycl::device &D1, const sycl::device &D2) {
                std::hash<sycl::device> Hasher;
                return Hasher(D1) < Hasher(D2);
              });
    auto NewEnd = std::unique(Res.begin(), Res.end());
    Res.erase(NewEnd, Res.end());
    return Res;
  }

  ur_device_type_t UrDeviceType = [DeviceType]() {
    switch (DeviceType) {
    case info::device_type::all:
      return UR_DEVICE_TYPE_ALL;
    case info::device_type::gpu:
      return UR_DEVICE_TYPE_GPU;
    case info::device_type::cpu:
      return UR_DEVICE_TYPE_CPU;
    case info::device_type::accelerator:
      return UR_DEVICE_TYPE_FPGA;
    case info::device_type::automatic:
      return UR_DEVICE_TYPE_DEFAULT;
    default:
      throw sycl::exception(sycl::make_error_code(sycl::errc::invalid),
                            "Unknown device type.");
    }
  }();
  getDevicesImplHelper(UrDeviceType, Res);
  return Res;
}

void platform_impl::getDevicesImplHelper(ur_device_type_t UrDeviceType,
                                         std::vector<device> &OutVec) const {
  size_t InitialOutVecSize = OutVec.size();

  uint32_t NumDevices = 0;
  MAdapter->call<UrApiKind::urDeviceGet>(MPlatform, UrDeviceType,
                                         0u, // CP info::device_type::all
                                         nullptr, &NumDevices);

  if (NumDevices == 0) {
    // If platform doesn't have devices (even without filter)
    // LastDeviceIds[PlatformId] stay 0 that affects next platform devices num
    // analysis. Doing adjustment by simple copy of last device num from
    // previous platform.
    // Needs non const adapter reference.
    std::vector<adapter_impl *> &Adapters = ur::initializeUr();
    auto It = std::find_if(Adapters.begin(), Adapters.end(),
                           [&Platform = MPlatform](adapter_impl *&Adapter) {
                             return Adapter->containsUrPlatform(Platform);
                           });
    if (It != Adapters.end()) {
      adapter_impl *&Adapter = *It;
      std::lock_guard<std::mutex> Guard(*Adapter->getAdapterMutex());
      Adapter->adjustLastDeviceId(MPlatform);
    }
    return;
  }

  std::vector<ur_device_handle_t> UrDevices(NumDevices);
  // TODO catch an exception and put it to list of asynchronous exceptions
  MAdapter->call<UrApiKind::urDeviceGet>(
      MPlatform,
      UrDeviceType, // CP info::device_type::all
      NumDevices, UrDevices.data(), nullptr);

  // Some elements of UrDevices vector might be filtered out, so make a copy of
  // handles to do a cleanup later
  std::vector<ur_device_handle_t> UrDevicesToCleanUp = UrDevices;

  // Filter out devices that are not present in the SYCL_DEVICE_ALLOWLIST
  if (SYCLConfig<SYCL_DEVICE_ALLOWLIST>::get())
    applyAllowList(UrDevices, MPlatform, *MAdapter);

  ods_target_list *OdsTargetList = SYCLConfig<ONEAPI_DEVICE_SELECTOR>::get();

  // The first step is to filter out devices that are not compatible with
  // ONEAPI_DEVICE_SELECTOR. This is also the mechanism by which top level
  // device ids are assigned.
  std::vector<int> PlatformDeviceIndices;
  if (OdsTargetList) {
    PlatformDeviceIndices = filterDeviceFilter<ods_target_list, ods_target>(
        UrDevices, OdsTargetList);
  }

  // The next step is to inflate the filtered UrDevices into SYCL Device
  // objects.
  platform_impl &PlatformImpl = getOrMakePlatformImpl(MPlatform, *MAdapter);
  std::transform(UrDevices.begin(), UrDevices.end(), std::back_inserter(OutVec),
                 [&PlatformImpl](const ur_device_handle_t UrDevice) -> device {
                   return detail::createSyclObjFromImpl<device>(
                       PlatformImpl.getOrMakeDeviceImpl(UrDevice));
                 });

  // The reference counter for handles, that we used to create sycl objects, is
  // incremented, so we need to call release here.
  for (ur_device_handle_t &UrDev : UrDevicesToCleanUp)
    MAdapter->call<UrApiKind::urDeviceRelease>(UrDev);

  // If we aren't using ONEAPI_DEVICE_SELECTOR, then we are done.
  // and if there are no new devices, there won't be any need to replace them
  // with subdevices.
  if (!OdsTargetList || OutVec.size() == InitialOutVecSize)
    return;

  // Otherwise, our last step is to revisit the devices, possibly replacing
  // them with subdevices (which have been ignored until now)
  OutVec = amendDeviceAndSubDevices(getBackend(), OutVec, OdsTargetList,
                                    PlatformDeviceIndices, PlatformImpl);
}

bool platform_impl::has_extension(const std::string &ExtensionName) const {

  std::string AllExtensionNames = urGetInfoString<UrApiKind::urPlatformGetInfo>(
      *this, detail::UrInfoCode<info::platform::extensions>::value);
  return (AllExtensionNames.find(ExtensionName) != std::string::npos);
}

bool platform_impl::supports_usm() const {
  return getBackend() != backend::opencl ||
         has_extension("cl_intel_unified_shared_memory");
}

ur_native_handle_t platform_impl::getNative() const {
  adapter_impl &Adapter = getAdapter();
  ur_native_handle_t Handle = 0;
  Adapter.call<UrApiKind::urPlatformGetNativeHandle>(getHandleRef(), &Handle);
  return Handle;
}

device select_device(DSelectorInvocableType DeviceSelectorInvocable,
                     std::vector<device> &Devices);

// All devices on the platform must have the given aspect.
bool platform_impl::has(aspect Aspect) const {
  for (const auto &dev : get_devices()) {
    if (dev.has(Aspect) == false) {
      return false;
    }
  }
  return true;
}

device_impl *platform_impl::getDeviceImplHelper(ur_device_handle_t UrDevice) {
  for (const std::unique_ptr<device_impl> &Device : MDevices) {
    if (Device->getHandleRef() == UrDevice)
      return Device.get();
  }
  return nullptr;
}

} // namespace detail
} // namespace _V1
} // namespace sycl