feature: add umonitor and umwait synchronization function

Related-To: NEO-9737

Signed-off-by: Zbigniew Zdanowicz <[email protected]>

Author: zzdanowicz

level_zero/core/test/unit_tests/fixtures/event_fixture.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2023 Intel Corporation
+ * Copyright (C) 2023-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -20,6 +20,8 @@ template struct EventImp<uint64_t>;
 namespace ult {
 
 void EventFixtureImpl::setUpImpl(int32_t eventPoolHostFlag, int32_t eventPoolTimestampFlag) {
+    NEO::debugManager.flags.EnableWaitpkg.set(0);
+
     DeviceFixture::setUp();
 
     ze_event_pool_flags_t eventPoolFlags = 0;

level_zero/core/test/unit_tests/sources/fence/test_fence.cpp

@@ -1,12 +1,13 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
  */
 
 #include "shared/source/built_ins/sip.h"
 #include "shared/source/helpers/completion_stamp.h"
+#include "shared/source/utilities/wait_util.h"
 #include "shared/test/common/mocks/mock_command_stream_receiver.h"
 #include "shared/test/common/mocks/mock_csr.h"
 #include "shared/test/common/mocks/mock_device.h"
@@ -239,6 +240,9 @@ TEST_F(FenceSynchronizeTest, givenInfiniteTimeoutWhenWaitingForFenceCompletionTh
     constexpr uint32_t activePartitions = 2;
     constexpr uint32_t postSyncOffset = 16;
 
+    VariableBackup<bool> backupWaitpkgUse(&NEO::WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&NEO::WaitUtils::waitCount, 1);
+
     const auto csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
     ASSERT_NE(nullptr, csr->getTagAddress());
     csr->immWritePostSyncWriteOffset = postSyncOffset;

opencl/test/unit_test/api/cl_enqueue_unmap_mem_object_tests.inl

@@ -1,10 +1,13 @@
 /*
- * Copyright (C) 2018-2023 Intel Corporation
+ * Copyright (C) 2018-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
  */
 
+#include "shared/source/utilities/wait_util.h"
+#include "shared/test/common/helpers/variable_backup.h"
+
 #include "opencl/source/command_queue/command_queue.h"
 #include "opencl/source/sharings/sharing.h"
 #include "opencl/test/unit_test/api/cl_api_tests.h"
@@ -79,6 +82,8 @@ TEST_F(ClEnqueueUnmapMemObjTests, givenInvalidAddressWhenUnmappingOnCpuThenRetur
 TEST_F(ClEnqueueUnmapMemObjTests, givenZeroCopyWithoutCoherencyAllowedWhenMapAndUnmapThenFlushCachelines) {
     DebugManagerStateRestore restorer;
     debugManager.flags.AllowZeroCopyWithoutCoherency.set(1);
+    VariableBackup<bool> backupWaitpkgUse(&WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount, 1);
 
     auto buffer = std::unique_ptr<Buffer>(BufferHelper<BufferAllocHostPtr<>>::create(pContext));
     EXPECT_TRUE(buffer->mappingOnCpuAllowed());

opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_4_tests.cpp

@@ -1,12 +1,13 @@
 /*
- * Copyright (C) 2018-2023 Intel Corporation
+ * Copyright (C) 2018-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
  */
 
 #include "shared/source/command_stream/wait_status.h"
 #include "shared/source/helpers/timestamp_packet.h"
+#include "shared/source/utilities/wait_util.h"
 #include "shared/test/common/libult/ult_command_stream_receiver.h"
 #include "shared/test/common/mocks/mock_command_stream_receiver.h"
 #include "shared/test/common/mocks/mock_timestamp_container.h"
@@ -802,6 +803,8 @@ extern TaskCountType pauseValue;
 HWTEST_F(CommandStreamReceiverFlushTaskTests, givenTagValueNotMeetingTaskCountToWaitWhenTagValueSwitchesThenWaitFunctionReturnsTrue) {
     VariableBackup<volatile TagAddressType *> backupPauseAddress(&CpuIntrinsicsTests::pauseAddress);
     VariableBackup<TaskCountType> backupPauseValue(&CpuIntrinsicsTests::pauseValue);
+    VariableBackup<bool> backupWaitpkgUse(&WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount, 1);
 
     auto mockCsr = new MockCsrHw2<FamilyType>(*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield());
     pDevice->resetCommandStreamReceiver(mockCsr);
@@ -820,6 +823,8 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenTagValueNotMeetingTaskCountTo
 HWTEST_F(CommandStreamReceiverFlushTaskTests, givenTagValueNotMeetingTaskCountToWaitAndIndefinitelyPollWhenWaitForCompletionThenDoNotCallWaitUtils) {
     VariableBackup<volatile TagAddressType *> backupPauseAddress(&CpuIntrinsicsTests::pauseAddress);
     VariableBackup<TaskCountType> backupPauseValue(&CpuIntrinsicsTests::pauseValue);
+    VariableBackup<bool> backupWaitpkgUse(&WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount, 1);
 
     auto mockCsr = new MockCsrHw2<FamilyType>(*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield());
     pDevice->resetCommandStreamReceiver(mockCsr);

opencl/test/unit_test/helpers/timestamp_packet_1_tests.cpp

@@ -8,6 +8,7 @@
 #include "shared/source/helpers/gfx_core_helper.h"
 #include "shared/source/helpers/timestamp_packet.h"
 #include "shared/source/utilities/tag_allocator.h"
+#include "shared/source/utilities/wait_util.h"
 #include "shared/test/common/cmd_parse/hw_parse.h"
 #include "shared/test/common/helpers/dispatch_flags_helper.h"
 #include "shared/test/common/mocks/mock_csr.h"
@@ -1071,6 +1072,9 @@ extern std::function<void()> setupPauseAddress;
 } // namespace CpuIntrinsicsTests
 
 HWTEST_F(TimestampPacketTests, givenEnableTimestampWaitForQueuesWhenFinishThenCallWaitUtils) {
+    VariableBackup<bool> backupWaitpkgUse(&WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount, 1);
+
     DebugManagerStateRestore restorer;
     debugManager.flags.UpdateTaskCountFromWait.set(3);
     debugManager.flags.EnableTimestampWaitForQueues.set(1);

shared/source/utilities/wait_util.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2023 Intel Corporation
+ * Copyright (C) 2021-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -18,8 +18,8 @@ constexpr uint64_t defaultCounterValue = 10000;
 constexpr uint32_t defaultControlValue = 0;
 constexpr bool defaultEnableWaitPkg = false;
 
-uint64_t counterValue = defaultCounterValue;
-uint32_t controlValue = defaultControlValue;
+uint64_t waitpkgCounterValue = defaultCounterValue;
+uint32_t waitpkgControlValue = defaultControlValue;
 
 uint32_t waitCount = defaultWaitCount;
 
@@ -45,11 +45,11 @@ void init() {
 
     int64_t overrideWaitPkgCounter = debugManager.flags.WaitpkgCounterValue.get();
     if (overrideWaitPkgCounter != -1) {
-        counterValue = static_cast<uint64_t>(overrideWaitPkgCounter);
+        waitpkgCounterValue = static_cast<uint64_t>(overrideWaitPkgCounter);
     }
     int32_t overrideWaitPkgControl = debugManager.flags.WaitpkgControlValue.get();
     if (overrideWaitPkgControl != -1) {
-        controlValue = static_cast<uint32_t>(overrideWaitPkgControl);
+        waitpkgControlValue = static_cast<uint32_t>(overrideWaitPkgControl);
     }
 
     int32_t overrideWaitCount = debugManager.flags.WaitLoopCount.get();

shared/source/utilities/wait_util.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2023 Intel Corporation
+ * Copyright (C) 2021-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -19,12 +19,21 @@ namespace WaitUtils {
 
 constexpr uint32_t defaultWaitCount = 1u;
 
-extern uint64_t counterValue;
-extern uint32_t controlValue;
+extern uint64_t waitpkgCounterValue;
+extern uint32_t waitpkgControlValue;
 extern uint32_t waitCount;
 extern bool waitpkgSupport;
 extern bool waitpkgUse;
 
+inline bool monitorWait(volatile void const *monitorAddress, uint64_t counterModifier) {
+    uint64_t currentCounter = CpuIntrinsics::rdtsc();
+    currentCounter += (waitpkgCounterValue + counterModifier);
+
+    CpuIntrinsics::umonitor(const_cast<void *>(monitorAddress));
+    bool result = CpuIntrinsics::umwait(waitpkgControlValue, currentCounter) == 0;
+    return result;
+}
+
 template <typename T>
 inline bool waitFunctionWithPredicate(volatile T const *pollAddress, T expectedValue, std::function<bool(T, T)> predicate) {
     for (uint32_t i = 0; i < waitCount; i++) {
@@ -34,6 +43,13 @@ inline bool waitFunctionWithPredicate(volatile T const *pollAddress, T expectedV
         if (predicate(*pollAddress, expectedValue)) {
             return true;
         }
+        if (waitpkgUse) {
+            if (monitorWait(pollAddress, 0)) {
+                if (predicate(*pollAddress, expectedValue)) {
+                    return true;
+                }
+            }
+        }
     }
     std::this_thread::yield();
     return false;

shared/test/common/utilities/cpuintrinsics.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -37,7 +37,7 @@ uint64_t rdtscRetValue = 0;
 unsigned char umwaitRetValue = 0;
 
 std::function<void()> setupPauseAddress;
-std::function<unsigned char()> controlUmwait;
+std::function<void()> controlUmwait;
 } // namespace CpuIntrinsicsTests
 
 namespace NEO {
@@ -73,7 +73,8 @@ unsigned char umwait(unsigned int ctrl, uint64_t counter) {
     CpuIntrinsicsTests::lastUmwaitCounter = counter;
     CpuIntrinsicsTests::umwaitCounter++;
     if (CpuIntrinsicsTests::controlUmwait) {
-        return CpuIntrinsicsTests::controlUmwait();
+        CpuIntrinsicsTests::controlUmwait();
+        return CpuIntrinsicsTests::umwaitRetValue;
     } else {
         return CpuIntrinsicsTests::umwaitRetValue;
     }

shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp

@@ -1763,6 +1763,9 @@ extern uint32_t pauseOffset;
 } // namespace CpuIntrinsicsTests
 
 TEST(CommandStreamReceiverSimpleTest, givenMultipleActivePartitionsWhenWaitingForTaskCountForCleaningTemporaryAllocationsThenExpectAllPartitionTaskCountsAreChecked) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableWaitpkg.set(0);
+
     MockExecutionEnvironment executionEnvironment;
     executionEnvironment.prepareRootDeviceEnvironments(1);
     executionEnvironment.initializeMemoryManager();
@@ -1807,6 +1810,9 @@ TEST(CommandStreamReceiverSimpleTest, givenMultipleActivePartitionsWhenWaitingFo
 }
 
 TEST(CommandStreamReceiverSimpleTest, givenEmptyTemporaryAllocationListWhenWaitingForTaskCountForCleaningTemporaryAllocationsThenDoNotWait) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableWaitpkg.set(0);
+
     MockExecutionEnvironment executionEnvironment;
     executionEnvironment.prepareRootDeviceEnvironments(1);
     executionEnvironment.initializeMemoryManager();

shared/test/unit_test/direct_submission/linux/drm_direct_submission_tests.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -14,6 +14,7 @@
 #include "shared/source/os_interface/linux/drm_gem_close_worker.h"
 #include "shared/source/os_interface/linux/os_context_linux.h"
 #include "shared/source/os_interface/linux/sys_calls.h"
+#include "shared/source/utilities/wait_util.h"
 #include "shared/test/common/cmd_parse/hw_parse.h"
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
 #include "shared/test/common/helpers/engine_descriptor_helper.h"
@@ -794,6 +795,9 @@ HWTEST_F(DrmDirectSubmissionTest, givenDirectSubmissionNewResourceTlbFlushZeroAn
 HWCMDTEST_F(IGFX_XE_HP_CORE, DrmDirectSubmissionTest, givenMultipleActiveTilesWhenWaitingForTagUpdateThenQueryAllActiveTiles) {
     using Dispatcher = RenderDispatcher<FamilyType>;
 
+    VariableBackup<bool> backupWaitpkgUse(&WaitUtils::waitpkgUse, false);
+    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount, 1);
+
     MockDrmDirectSubmission<FamilyType, Dispatcher> directSubmission(*device->getDefaultEngine().commandStreamReceiver);
 
     uint32_t offset = directSubmission.immWritePostSyncOffset;

shared/test/unit_test/utilities/wait_util_tests.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2023 Intel Corporation
+ * Copyright (C) 2021-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -18,7 +18,21 @@ namespace CpuIntrinsicsTests {
 extern std::atomic<uint32_t> pauseCounter;
 } // namespace CpuIntrinsicsTests
 
-TEST(WaitTest, givenDefaultSettingsWhenNoPollAddressProvidedThenPauseDefaultTimeAndReturnFalse) {
+struct WaitPredicateOnlyFixture {
+    void setUp() {
+        debugManager.flags.EnableWaitpkg.set(0);
+        backupWaitCount = std::make_unique<VariableBackup<uint32_t>>(&WaitUtils::waitCount);
+    }
+
+    void tearDown() {}
+
+    DebugManagerStateRestore restore;
+    std::unique_ptr<VariableBackup<uint32_t>> backupWaitCount;
+};
+
+using WaitPredicateOnlyTest = Test<WaitPredicateOnlyFixture>;
+
+TEST_F(WaitPredicateOnlyTest, givenDefaultSettingsWhenNoPollAddressProvidedThenPauseDefaultTimeAndReturnFalse) {
     EXPECT_EQ(1u, WaitUtils::defaultWaitCount);
 
     WaitUtils::init();
@@ -30,10 +44,7 @@ TEST(WaitTest, givenDefaultSettingsWhenNoPollAddressProvidedThenPauseDefaultTime
     EXPECT_EQ(oldCount + WaitUtils::waitCount, CpuIntrinsicsTests::pauseCounter);
 }
 
-TEST(WaitTest, givenDebugFlagOverridesWhenNoPollAddressProvidedThenPauseDefaultTimeAndReturnFalse) {
-    DebugManagerStateRestore restore;
-    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount);
-
+TEST_F(WaitPredicateOnlyTest, givenDebugFlagOverridesWhenNoPollAddressProvidedThenPauseDefaultTimeAndReturnFalse) {
     uint32_t count = 10u;
     debugManager.flags.WaitLoopCount.set(count);
 
@@ -46,7 +57,7 @@ TEST(WaitTest, givenDebugFlagOverridesWhenNoPollAddressProvidedThenPauseDefaultT
     EXPECT_EQ(oldCount + count, CpuIntrinsicsTests::pauseCounter);
 }
 
-TEST(WaitTest, givenDefaultSettingsWhenPollAddressProvidedDoesNotMeetCriteriaThenPauseDefaultTimeAndReturnFalse) {
+TEST_F(WaitPredicateOnlyTest, givenDefaultSettingsWhenPollAddressProvidedDoesNotMeetCriteriaThenPauseDefaultTimeAndReturnFalse) {
     WaitUtils::init();
     EXPECT_EQ(WaitUtils::defaultWaitCount, WaitUtils::waitCount);
 
@@ -59,7 +70,7 @@ TEST(WaitTest, givenDefaultSettingsWhenPollAddressProvidedDoesNotMeetCriteriaThe
     EXPECT_EQ(oldCount + WaitUtils::waitCount, CpuIntrinsicsTests::pauseCounter);
 }
 
-TEST(WaitTest, givenDefaultSettingsWhenPollAddressProvidedMeetsCriteriaThenPauseDefaultTimeAndReturnTrue) {
+TEST_F(WaitPredicateOnlyTest, givenDefaultSettingsWhenPollAddressProvidedMeetsCriteriaThenPauseDefaultTimeAndReturnTrue) {
     WaitUtils::init();
     EXPECT_EQ(WaitUtils::defaultWaitCount, WaitUtils::waitCount);
 
@@ -72,10 +83,7 @@ TEST(WaitTest, givenDefaultSettingsWhenPollAddressProvidedMeetsCriteriaThenPause
     EXPECT_EQ(oldCount + WaitUtils::waitCount, CpuIntrinsicsTests::pauseCounter);
 }
 
-TEST(WaitTest, givenDebugFlagSetZeroWhenPollAddressProvidedMeetsCriteriaThenPauseZeroTimesAndReturnTrue) {
-    DebugManagerStateRestore restore;
-    VariableBackup<uint32_t> backupWaitCount(&WaitUtils::waitCount);
-
+TEST_F(WaitPredicateOnlyTest, givenDebugFlagSetZeroWhenPollAddressProvidedMeetsCriteriaThenPauseZeroTimesAndReturnTrue) {
     uint32_t count = 0u;
     debugManager.flags.WaitLoopCount.set(count);