fix: align NEO to new Xe KMD header

Align to the new PAT and cache coherency support

There is an issue with coherency=non_coh, which
is default option for some platforms.
Add temporary W/A until this issue is resolved.

xe_drm.h header is generated from the series
"PAT and cache coherency support"
from https://patchwork.freedesktop.org/series/123027/

Related-To: NEO-9421, NEO-8324
Signed-off-by: Naklicki, Mateusz <[email protected]>

Author: luluu9

shared/source/debug_settings/debug_variables_base.inl

@@ -267,6 +267,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, EnableInOrderRelaxedOrderingForEventsChaining, -
 DECLARE_DEBUG_VARIABLE(int32_t, InOrderAtomicSignallingEnabled, -1, "-1: default, 0: disabled, 1: Use atomic GPU operations in increment the counter. Otherwise use non-atomic commands like SDI.")
 DECLARE_DEBUG_VARIABLE(int32_t, InOrderDuplicatedCounterStorageEnabled, -1, "-1: default, 0: disabled, 1: Allocate additional host storage for signalling")
 DECLARE_DEBUG_VARIABLE(int32_t, SetProcessPowerThrottlingState, -1, "-1: default, 0: Disabled, 1: ECO, 2: HIGH. If set, will override process power throttling state on os context init. Windows only.")
+DECLARE_DEBUG_VARIABLE(int32_t, OverrideCpuCaching, -1, "-1: default, 1: DRM_XE_GEM_CPU_CACHING_WB, 2: DRM_XE_GEM_CPU_CACHING_WC")
 
 /*LOGGING FLAGS*/
 DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level")

shared/source/os_interface/linux/drm_neo.cpp

@@ -1315,6 +1315,7 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI
             UNRECOVERABLE_IF(bo->peekPatIndex() == CommonConstants::unsupportedPatIndex);
             ioctlHelper->fillVmBindExtSetPat(vmBindExtSetPat, bo->peekPatIndex(), castToUint64(extensions.get()));
             vmBind.extensions = castToUint64(vmBindExtSetPat);
+            vmBind.patIndex = bo->peekPatIndex();
         } else {
             vmBind.extensions = castToUint64(extensions.get());
         }

shared/source/os_interface/linux/ioctl_helper.h

@@ -61,6 +61,7 @@ struct VmBindParams {
     uint64_t length;
     uint64_t flags;
     uint64_t extensions;
+    uint64_t patIndex;
 };
 
 struct UuidRegisterResult {

shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp

@@ -533,6 +533,15 @@ void IoctlHelperXe::setDefaultEngine() {
     }
 }
 
+uint16_t IoctlHelperXe::getCpuCachingMode() {
+    uint16_t cpuCachingMode = DRM_XE_GEM_CPU_CACHING_WC;
+    if (debugManager.flags.OverrideCpuCaching.get() != -1) {
+        cpuCachingMode = debugManager.flags.OverrideCpuCaching.get();
+    }
+
+    return cpuCachingMode;
+}
+
 int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional<uint32_t> vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) {
     struct drm_xe_gem_create create = {};
     uint32_t regionsSize = static_cast<uint32_t>(memClassInstances.size());
@@ -553,6 +562,7 @@ int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t a
         memoryInstances.set(memoryClassInstance.memoryInstance);
     }
     create.flags = static_cast<uint32_t>(memoryInstances.to_ulong());
+    create.cpu_caching = this->getCpuCachingMode();
 
     auto ret = IoctlHelper::ioctl(DrmIoctl::gemCreate, &create);
     handle = create.handle;
@@ -586,6 +596,7 @@ uint32_t IoctlHelperXe::createGem(uint64_t size, uint32_t memoryBanks) {
         memoryInstances.set(regionClassAndInstance.memoryInstance);
     }
     create.flags = static_cast<uint32_t>(memoryInstances.to_ulong());
+    create.cpu_caching = this->getCpuCachingMode();
     [[maybe_unused]] auto ret = ioctl(DrmIoctl::gemCreate, &create);
     DEBUG_BREAK_IF(ret != 0);
     updateBindInfo(create.handle, 0u, create.size);
@@ -1254,6 +1265,7 @@ int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) {
         bind.bind.addr = gmmHelper->decanonize(vmBindParams.start);
         bind.bind.flags = DRM_XE_VM_BIND_FLAG_ASYNC;
         bind.bind.obj_offset = vmBindParams.offset;
+        bind.bind.pat_index = vmBindParams.patIndex;
 
         if (isBind) {
             bind.bind.op = DRM_XE_VM_BIND_OP_MAP;

shared/source/os_interface/linux/xe/ioctl_helper_xe.h

@@ -119,6 +119,7 @@ class IoctlHelperXe : public IoctlHelper {
     void fillBindInfoForIpcHandle(uint32_t handle, size_t size) override;
     bool getFdFromVmExport(uint32_t vmId, uint32_t flags, int32_t *fd) override;
     bool isImmediateVmBindRequired() const override;
+    uint16_t getCpuCachingMode();
 
   private:
     template <typename... XeLogArgs>

shared/test/common/test_files/igdrcl.config

@@ -572,4 +572,5 @@ EnableDeviceStateVerificationAfterFailedSubmission = -1
 InOrderAtomicSignallingEnabled = -1
 SetProcessPowerThrottlingState = -1
 InOrderDuplicatedCounterStorageEnabled = -1
+OverrideCpuCaching = -1
 # Please don't edit below this line

shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp

@@ -45,7 +45,7 @@ TEST(IoctlHelperXeTest, whenChangingBufferBindingThenWaitIsNeededAlways) {
 
 TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsThenDummyValueIsReturned) {
     auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
-    DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]};
+    DrmMockXe drm{*executionEnvironment->rootDeviceEnvironments[0]};
     auto xeIoctlHelper = std::make_unique<MockIoctlHelperXe>(drm);
     ASSERT_NE(nullptr, xeIoctlHelper);
 
@@ -62,11 +62,12 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsThen
     EXPECT_TRUE(xeIoctlHelper->bindInfo.empty());
     EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, {}, -1, false, numOfChunks));
     EXPECT_FALSE(xeIoctlHelper->bindInfo.empty());
+    EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching);
 }
 
 TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsAndVmIdThenDummyValueIsReturned) {
     auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
-    DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]};
+    DrmMockXe drm{*executionEnvironment->rootDeviceEnvironments[0]};
     auto xeIoctlHelper = std::make_unique<MockIoctlHelperXe>(drm);
     ASSERT_NE(nullptr, xeIoctlHelper);
 
@@ -84,6 +85,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsAndV
     EXPECT_TRUE(xeIoctlHelper->bindInfo.empty());
     EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, test.vmId, -1, false, numOfChunks));
     EXPECT_FALSE(xeIoctlHelper->bindInfo.empty());
+    EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching);
 }
 
 TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGemCreateAndNoLocalMemoryThenProperValuesSet) {
@@ -107,6 +109,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGemCreateAndNoLocalMemoryThenP
 
     EXPECT_EQ(size, drm.createParamsSize);
     EXPECT_EQ(1u, drm.createParamsFlags);
+    EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching);
 
     // dummy mock handle
     EXPECT_EQ(handle, drm.createParamsHandle);
@@ -134,6 +137,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGemCreateWhenMemoryBanksZeroTh
 
     EXPECT_EQ(size, drm.createParamsSize);
     EXPECT_EQ(1u, drm.createParamsFlags);
+    EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching);
 
     // dummy mock handle
     EXPECT_EQ(handle, drm.createParamsHandle);
@@ -161,6 +165,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGemCreateAndLocalMemoryThenPro
 
     EXPECT_EQ(size, drm.createParamsSize);
     EXPECT_EQ(6u, drm.createParamsFlags);
+    EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching);
 
     // dummy mock handle
     EXPECT_EQ(handle, drm.createParamsHandle);
@@ -583,6 +588,7 @@ TEST(IoctlHelperXeTest, whenCallingIoctlThenProperValueIsReturned) {
         test.handle = 0;
         test.flags = 1;
         test.size = 123;
+        test.cpu_caching = DRM_XE_GEM_CPU_CACHING_WC;
         ret = mockXeIoctlHelper->ioctl(DrmIoctl::gemCreate, &test);
         EXPECT_EQ(0, ret);
     }
@@ -1649,3 +1655,51 @@ TEST(IoctlHelperXeTest, givenXeIoctlHelperWhenInitializeGetGpuTimeFunctionIsCall
     xeIoctlHelper->initializeGetGpuTimeFunction();
     EXPECT_EQ(xeIoctlHelper->getGpuTime, nullptr);
 }
+
+TEST(IoctlHelperXeTest, givenIoctlHelperXeAndDebugOverrideEnabledWhenGetCpuCachingModeCalledThenOverriddenValueIsReturned) {
+    DebugManagerStateRestore restorer;
+    auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
+    DrmMockXe drm{*executionEnvironment->rootDeviceEnvironments[0]};
+
+    auto xeIoctlHelper = std::make_unique<MockIoctlHelperXe>(drm);
+    drm.memoryInfo.reset(xeIoctlHelper->createMemoryInfo().release());
+    ASSERT_NE(nullptr, xeIoctlHelper);
+
+    debugManager.flags.OverrideCpuCaching.set(DRM_XE_GEM_CPU_CACHING_WB);
+    EXPECT_EQ(xeIoctlHelper->getCpuCachingMode(), DRM_XE_GEM_CPU_CACHING_WB);
+
+    debugManager.flags.OverrideCpuCaching.set(DRM_XE_GEM_CPU_CACHING_WC);
+    EXPECT_EQ(xeIoctlHelper->getCpuCachingMode(), DRM_XE_GEM_CPU_CACHING_WC);
+}
+
+TEST(IoctlHelperXeTest, whenCallingVmBindThenPatIndexIsSet) {
+    DebugManagerStateRestore restorer;
+    auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
+    DrmMockXe drm{*executionEnvironment->rootDeviceEnvironments[0]};
+    auto xeIoctlHelper = std::make_unique<MockIoctlHelperXe>(drm);
+
+    uint64_t fenceAddress = 0x4321;
+    uint64_t fenceValue = 0x789;
+    uint64_t expectedPatIndex = 0xba;
+
+    BindInfo mockBindInfo{};
+    mockBindInfo.handle = 0x1234;
+    xeIoctlHelper->bindInfo.push_back(mockBindInfo);
+
+    VmBindExtUserFenceT vmBindExtUserFence{};
+
+    xeIoctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, fenceAddress, fenceValue, 0u);
+
+    VmBindParams vmBindParams{};
+    vmBindParams.handle = mockBindInfo.handle;
+    vmBindParams.extensions = castToUint64(&vmBindExtUserFence);
+    vmBindParams.patIndex = expectedPatIndex;
+
+    drm.vmBindInputs.clear();
+    drm.syncInputs.clear();
+    drm.waitUserFenceInputs.clear();
+    ASSERT_EQ(0, xeIoctlHelper->vmBind(vmBindParams));
+    ASSERT_EQ(1u, drm.vmBindInputs.size());
+
+    EXPECT_EQ(drm.vmBindInputs[0].bind.pat_index, expectedPatIndex);
+}

shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.h

@@ -150,7 +150,8 @@ class DrmMockXe : public DrmMockCustom {
             this->createParamsSize = createParams->size;
             this->createParamsFlags = createParams->flags;
             this->createParamsHandle = createParams->handle = testValueGemCreate;
-            if (0 == this->createParamsSize || 0 == this->createParamsFlags) {
+            this->createParamsCpuCaching = createParams->cpu_caching;
+            if (0 == this->createParamsSize || 0 == this->createParamsFlags || 0 == this->createParamsCpuCaching) {
                 return EINVAL;
             }
             ret = 0;
@@ -268,5 +269,6 @@ class DrmMockXe : public DrmMockCustom {
     StackVec<drm_xe_sync, 1> syncInputs;
     int waitUserFenceReturn = 0;
     uint32_t createParamsFlags = 0u;
+    uint16_t createParamsCpuCaching = 0u;
     bool ioctlCalled = false;
 };