Fix tests

.github/workflows/run-ui-tests.yml

@@ -37,12 +37,12 @@ jobs:
 
   testUI:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 30
+    timeout-minutes: 60
     needs: getPlugin
     strategy:
       fail-fast: false
       matrix:
-        os: [ macos-latest, ubuntu-latest, windows-latest ]
+        os: [ macos-13, ubuntu-latest, windows-latest ]
         ideDate: # Up to 2 versions per year, initial major ver release date should be not older than a year
           #          - IC-LATEST_EAP # TODO: Run this variant weekly
           - IC-2023.3.2 # released on 20.12.2023, major from 06.12.2023
@@ -62,7 +62,7 @@ jobs:
               sleep 10
               ./gradlew :plugin-test:test
             reportName: ui-tests-linux
-          - os: macos-latest
+          - os: macos-13
             runTests: |
               ./gradlew :plugin-test:test
             reportName: ui-tests-mac
@@ -83,13 +83,21 @@ jobs:
         with:
           script: return "${{ matrix.ideDate }}".split("-", 2)[1]
           result-encoding: string
-      - name: Setup FFmpeg
+      - name: Setup FFmpeg via action
         uses: FedericoCarboni/setup-ffmpeg@v2
         with:
           # Not strictly necessary, but it may prevent rate limit
           # errors especially on GitHub-hosted macos machines.
           token: ${{ secrets.GITHUB_TOKEN }}
 
+# Uncomment when migrating to version 14
+#      - name: Install homebrew (for FFmpeg)
+#        uses: Homebrew/actions/setup-homebrew@e05416b42376bcda221f9102c4f595f4994016be
+#        if: ${{ startsWith(matrix.os,  'macos-') }}
+#      - name: Setup FFmpeg via homebrew
+#        if: ${{ startsWith(matrix.os,  'macos-') }}
+#        run: brew install ffmpeg
+
       # Setup Java environment for the next steps
       - name: Setup Java
         uses: actions/setup-java@v4

.gitignore

@@ -1,9 +1,12 @@
 .gradle
 .idea
 .qodana
+.jqwik-database
 build
+/plugin-test/screenshot
+/plugin-test/video
 
 # Generated with Nix Flakes:
 /result
 /ipredicate/project/src/main/kotlin/com/github/tarcv/testingteam/surveyor/ipredicate/GSPredicate.kt
-/ipredicate/project/src/main/kotlin/com/github/tarcv/testingteam/surveyor/ipredicate/WebDriverAgent.kt
+/ipredicate/project/src/main/kotlin/com/github/tarcv/testingteam/surveyor/ipredicate/WebDriverAgent.kt

plugin-test/src/test/kotlin/com/github/tarcv/testingteam/surveyoridea/ScreenshotTest.kt

@@ -0,0 +1,13 @@
+package com.github.tarcv.testingteam.surveyoridea
+
+import com.github.tarcv.testingteam.surveyoridea.gui.BaseTestProjectTests.Companion.REQUIRES_SCREENSHOT_ASSUMPTIONS_TAG
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Test
+import kotlin.annotation.AnnotationRetention.RUNTIME
+import kotlin.annotation.AnnotationTarget.FUNCTION
+
+@Retention(RUNTIME)
+@Target(FUNCTION)
+@Tag(REQUIRES_SCREENSHOT_ASSUMPTIONS_TAG)
+@Test
+annotation class ScreenshotTest

plugin-test/src/test/kotlin/com/github/tarcv/testingteam/surveyoridea/gui/BaseTestProjectTests.kt

@@ -13,6 +13,7 @@ import com.intellij.remoterobot.utils.WaitForConditionTimeoutException
 import com.intellij.remoterobot.utils.waitForIgnoringError
 import org.apache.commons.io.file.PathUtils
 import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.TestInfo
 import org.junit.jupiter.api.extension.ExtendWith
@@ -35,6 +36,12 @@ open class BaseTestProjectTests {
 
         @JvmStatic
         protected val editorWithSnapshot = "editorWithSnapshot"
+
+        const val REQUIRES_SCREENSHOT_ASSUMPTIONS_TAG = "REQUIRES_SCREENSHOT_ASSUMPTIONS"
+
+        fun checkScreenshotAssumptions(robot: RemoteRobot) {
+            Assumptions.assumeTrue(robot.isLinux(), "Screenshot are only enabled on Linux")
+        }
     }
 
     @field:TempDir
@@ -58,7 +65,11 @@ open class BaseTestProjectTests {
     protected val commonSteps by lazy { CommonSteps(remoteRobot) }
 
     @BeforeEach
-    fun openTestProject() {
+    fun openTestProject(testInfo: TestInfo) {
+        if (REQUIRES_SCREENSHOT_ASSUMPTIONS_TAG in testInfo.tags) {
+            checkScreenshotAssumptions(remoteRobot)
+        }
+
         PathUtils.cleanDirectory(tempDir)
 
         projectPath = tempDir.resolve("project")
@@ -71,11 +82,12 @@ open class BaseTestProjectTests {
                 ifTipOfTheDayDialogPresent {
                     close()
                 }
-
                 closeCodeWithMeBubbleIfNeeded()
-
-                resizeWindow(1280, 800)
                 commonSteps.waitForSmartMode(1)
+
+                if (REQUIRES_SCREENSHOT_ASSUMPTIONS_TAG in testInfo.tags) {
+                    resizeWindow(1024, 768)
+                }
             }
         }
     }
@@ -89,20 +101,6 @@ open class BaseTestProjectTests {
         }
     }
 
-    private fun IdeaFrame.resizeWindow(newWidth: Int, newHeight: Int) {
-        runJs(
-            runInEdt = true, script =
-            """
-                    importPackage(com.intellij.openapi.application)
-                    ApplicationManager.getApplication().invokeLater(new Runnable({
-                        run: function () {
-                            component.setSize($newWidth, $newHeight)
-                        }
-                    }))
-                          """.trimIndent()
-        )
-    }
-
     @AfterEach
     fun closeTestProject(testInfo: TestInfo): Unit = with(remoteRobot) {
         kotlin.runCatching {

plugin-test/src/test/kotlin/com/github/tarcv/testingteam/surveyoridea/gui/EditingTest.kt

@@ -1,14 +1,15 @@
 package com.github.tarcv.testingteam.surveyoridea.gui
 
+import com.github.tarcv.testingteam.surveyoridea.ScreenshotTest
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.idea
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.locateElementToolWindow
 import com.github.tarcv.testingteam.surveyoridea.hasJavaSupport
 import com.github.tarcv.testingteam.surveyoridea.waitingAssertEquals
-import com.intellij.remoterobot.client.IdeaSideException
 import com.intellij.remoterobot.fixtures.EditorFixture
 import com.intellij.remoterobot.utils.keyboard
 import com.intellij.remoterobot.utils.waitFor
 import org.apache.commons.text.StringEscapeUtils
+import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.api.Test
 import java.awt.Point
 import java.lang.Thread.sleep
@@ -26,6 +27,12 @@ class EditingTest : BaseTestProjectTests() {
         assertDroidSelectorEditing("""new UiSelector()${'\n'}.index(1).""", 2)
     }
 
+    @ScreenshotTest
+    fun screenshotDroidSelectorAutocomplete() {
+        Assumptions.assumeTrue(hasJavaSupport, "Java support is required for autocomplete list to appear")
+        assertDroidSelectorEditing("""new UiSelector().textContains("ICE").""", 1)
+    }
+
     private fun assertDroidSelectorEditing(selector: String, expectedLineCount: Int) = with(remoteRobot) {
         idea {
             openFileAndToolWindow(relativeToProject(droidAutomatorSnapshotFile), editorWithSnapshot)
@@ -71,21 +78,21 @@ class EditingTest : BaseTestProjectTests() {
 
                 if (hasJavaSupport) {
                     val popupItems = waitFor(Duration.ofSeconds(10), functionWithCondition = {
-                        val items: List<String> = try {
-                            editor.callJs(
-                                runInEdt = true, script = """
+                        val items: List<String> = editor.callJs(
+                            runInEdt = true, script = """
                         importPackage(com.intellij.codeInsight.lookup)
-                        const model = LookupManager.getActiveLookup(local.get('editor')).getList().getModel()
-                        const listItems = new ArrayList();
-                        for(let i = 0; i < model.size; ++i) { 
-                            listItems.add(model.getElementAt(i).toString())
+                        try {
+                            const model = LookupManager.getActiveLookup(local.get('editor')).getList().getModel()
+                            const listItems = new ArrayList();
+                            for(let i = 0; i < model.size; ++i) { 
+                                listItems.add(model.getElementAt(i).toString())
+                            }
+                            listItems
+                        } catch {
                         }
-                        listItems
+                            new ArrayList()
                     """.trimIndent()
-                            )
-                        } catch (e: IdeaSideException) {
-                            emptyList()
-                        }
+                        )
                         (items.isNotEmpty() && items.contains("resourceId")) to items
                     })
                     assert(popupItems.contains("index")) { "contains 'index'" }
@@ -111,5 +118,6 @@ private fun EditorFixture.getLineCount(): Int {
         runInEdt = true, script = """
             const document = local.get('document')
             document.getLineCount()
-                    """.trimIndent())
+                    """.trimIndent()
+    )
 }

plugin-test/src/test/kotlin/com/github/tarcv/testingteam/surveyoridea/gui/LocateActionUiTests.kt

@@ -1,14 +1,16 @@
 package com.github.tarcv.testingteam.surveyoridea.gui
 
+import com.github.tarcv.testingteam.surveyoridea.ScreenshotTest
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.IdeaFrame
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.LocateElementToolWindowFixture
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.idea
 import com.github.tarcv.testingteam.surveyoridea.gui.fixtures.locateElementToolWindow
 import com.github.tarcv.testingteam.surveyoridea.trimAllIndent
 import com.github.tarcv.testingteam.surveyoridea.waitingAssertEquals
-import com.intellij.remoterobot.client.IdeaSideException
+import com.intellij.remoterobot.fixtures.JPopupMenuFixture
 import com.intellij.remoterobot.utils.keyboard
 import org.apache.commons.text.StringEscapeUtils
+import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.MethodSource
@@ -40,16 +42,14 @@ class LocateActionUiTests : BaseTestProjectTests() {
                                         selected="false" bounds="[250,933][830,1057]"/>
                                     """.trimAllIndent()
             val singleLineSelector = """new UiSelector().resourceIdMatches(".+/celsiusText")"""
-            for (type in listOf(null, droidAutomatorSelectorType)) {
-                add(
-                    LocationSetupData(
-                        droidAutomatorSnapshotFile,
-                        type,
-                        singleLineSelector,
-                        expectedTag
-                    )
+            add(
+                LocationSetupData(
+                    droidAutomatorSnapshotFile,
+                    null,
+                    singleLineSelector,
+                    expectedTag
                 )
-            }
+            )
             add(
                 LocationSetupData(
                     droidAutomatorSnapshotFile,
@@ -189,6 +189,23 @@ class LocateActionUiTests : BaseTestProjectTests() {
             }
         }
 
+    @ScreenshotTest
+    fun screenshotLocatingFromToolButton() {
+        testLocatingFromToolButton(
+            LocationSetupData(
+                droidAutomatorSnapshotFile,
+                null,
+                """new UiSelector().textContains("ICE")""",
+                """
+                        <node index="1" text="ICE MELTING (0°C)" resource-id="" class="android.widget.Button"
+                            package="com.github.tarcv.converter" content-desc="" checkable="false" checked="false"
+                            clickable="true" enabled="true" focusable="true" focused="false" scrollable="false"
+                            long-clickable="false" password="false" selected="false" bounds="[618,501][750,897]"/>
+                """.trimAllIndent()
+            )
+        )
+    }
+
     @TestAllLocatorTypes
     fun testLocatingFromToolButton(locationSetupData: LocationSetupData) {
         verifyLocatingFrom(locationSetupData) {
@@ -198,17 +215,43 @@ class LocateActionUiTests : BaseTestProjectTests() {
         }
     }
 
+    @ScreenshotTest
+    fun screenshotLocatorTypes() = with(remoteRobot) {
+        Assumptions.assumeTrue(isLinux())
+        idea {
+            openFileAndToolWindow(relativeToProject(droidAutomatorSnapshotFile), editorWithSnapshot)
+            locateElementToolWindow {
+                locatorTypeDropdown.click()
+                sleep(5_000)
+            }
+        }
+    }
+
+    @ScreenshotTest
+    fun screenshotOpeningToolWindowFromMainMenu() = with(remoteRobot) {
+        Assumptions.assumeTrue(isLinux())
+        idea {
+            openFileInTestProject(relativeToProject(droidAutomatorSnapshotFile), editorWithSnapshot)
+            selectInMenuBar(
+                "View",
+                "Tool Windows"
+            )
+            remoteRobot.findAll<JPopupMenuFixture>(JPopupMenuFixture.byContainsItem("Locate Element"))
+                .first()
+                .menuItem("Locate Element")
+                .moveMouse()
+        }
+    }
+
     @Test
     fun testOpeningToolWindowFromMainMenu() = with(remoteRobot) {
         idea {
-            idea {
-                openFileInTestProject(relativeToProject(droidAutomatorSnapshotFile), editorWithSnapshot)
-                selectInMenuBar(
-                    "View",
-                    "Tool Windows",
-                    "Locate Element"
-                )
-            }
+            openFileInTestProject(relativeToProject(droidAutomatorSnapshotFile), editorWithSnapshot)
+            selectInMenuBar(
+                "View",
+                "Tool Windows",
+                "Locate Element"
+            )
             locateElementToolWindow {
                 editor
             }
@@ -249,15 +292,7 @@ class LocateActionUiTests : BaseTestProjectTests() {
                 "Correct node should be selected.",
                 locationSetupData.expectedTag
             ) {
-                try {
-                    getSelectedXmlNodeOuterXml(editorWithSnapshot).trimAllIndent()
-                } catch (e: IdeaSideException) {
-                    if (e.message?.contains("TypeError: Cannot read property \"editor\" from null") == true) {
-                        "<Couldn't get editor instance>"
-                    } else {
-                        throw e
-                    }
-                }
+                getSelectedXmlNodeOuterXml(editorWithSnapshot).trimAllIndent()
             }
         }
     }