test(expo): maestro improvements

.changeset/maestro-ci-hardening.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/expo-native-build.yml

@@ -28,6 +28,12 @@ env:
   E2E_INSTANCE_NAME: clerkstage-with-native-components
   BAPI_URL: https://api.clerkstage.dev
   MAESTRO_VERSION: '2.8.0'
+  # Skips a blocking network call on every CLI start.
+  MAESTRO_CLI_NO_ANALYTICS: '1'
+  # Devices per e2e job; the flows are split across them. A second driver
+  # took up to 130s to come up beside the first, past the default startup timeout.
+  MAESTRO_SHARDS: '2'
+  MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000'
 
 jobs:
   native-build:
@@ -66,6 +72,10 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Boot iOS simulators
+        if: matrix.platform == 'ios' && matrix.run-e2e == true
+        run: ${{ env.E2E_DIR }}/boot-ios-simulators.sh boot "$MAESTRO_SHARDS"
+
       - name: Compute native build cache key
         id: native-build-key
         env:
@@ -233,29 +243,17 @@ jobs:
       - name: Install maestro CLI
         if: steps.keys.outputs.pk != ''
         run: |
-          set -o pipefail
           if [ -x "$HOME/.maestro/bin/maestro" ]; then
             echo "Using cached Maestro"
           else
-            installed=0
-            for i in 1 2 3; do
-              if curl -fLs --retry 3 --retry-delay 5 "https://get.maestro.mobile.dev" | MAESTRO_VERSION="$MAESTRO_VERSION" bash; then
-                if [ -x "$HOME/.maestro/bin/maestro" ]; then installed=1; break; fi
-              fi
-              echo "Maestro install attempt $i failed (or binary missing); retrying"
-              sleep 5
-            done
-            [ "$installed" = 1 ] || { echo "::error::Maestro install failed after 3 attempts"; exit 1; }
+            archive="$RUNNER_TEMP/maestro.zip"
+            curl -fLs --retry 3 --retry-delay 5 -o "$archive" \
+              "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$MAESTRO_VERSION/maestro.zip"
+            unzip -q "$archive" -d "$RUNNER_TEMP/maestro-dist"
+            mv "$RUNNER_TEMP/maestro-dist/maestro" "$HOME/.maestro"
           fi
           echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
-          "$HOME/.maestro/bin/maestro" --version
-
-      - name: Boot iOS simulator
-        id: sim
-        if: matrix.platform == 'ios' && steps.keys.outputs.pk != ''
-        uses: futureware-tech/simulator-action@e89aa8f93d3aec35083ff49d2854d07f7186f7f5 # v5
-        with:
-          model: 'iPhone 17 Pro'
+          test "$("$HOME/.maestro/bin/maestro" --version)" = "$MAESTRO_VERSION"
 
       # Provisioned as late as possible so live credentials exist for the
       # shortest window.
@@ -293,35 +291,23 @@ jobs:
         env:
           CLERK_TEST_EMAIL: ${{ steps.user.outputs.email }}
           CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
-          SIM_UDID: ${{ steps.sim.outputs.udid }}
         run: |
-          echo "Using simulator $SIM_UDID"
-          # Kill animations + predictive keyboard: animations add latency to
-          # every tap; predictive text hijacks inputText targets.
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true
-          xcrun simctl spawn "$SIM_UDID" defaults write -g ApplePersistenceIgnoreState -bool YES || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.ContinuousPath -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCorrection -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.Prediction -bool NO || true
-          # The one-time keyboard tutorial sheets carry their own Continue
-          # button, which can hijack taps on the AuthView's Continue.
-          for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do
-            xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.preferences "$key" -bool YES || true
-          done
-          xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app
-          # Stream the app's console output into the debug artifact so a hang has
-          # actionable evidence (keychain/network errors) instead of just screenshots.
+          "$GITHUB_WORKSPACE/$E2E_DIR/boot-ios-simulators.sh" wait
           mkdir -p "$RUNNER_TEMP/maestro-debug"
-          xcrun simctl spawn "$SIM_UDID" log stream --style compact \
-            --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \
-            > "$RUNNER_TEMP/maestro-debug/sim-console.log" 2>&1 &
-          LOG_PID=$!
-          cd "$GITHUB_WORKSPACE/$E2E_DIR"
+          log_pids=()
+          for udid in ${MAESTRO_UDID//,/ }; do
+            xcrun simctl install "$udid" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app
+            # Stream the app's console output into the debug artifact so a hang
+            # has actionable evidence (keychain/network errors), not just screenshots.
+            xcrun simctl spawn "$udid" log stream --style compact \
+              --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \
+              > "$RUNNER_TEMP/maestro-debug/$udid-console.log" 2>&1 &
+            log_pids+=($!)
+          done
           rc=0
-          MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" ./run-flows.sh \
-            xcrun simctl terminate "$SIM_UDID" com.clerk.exponativebuildfixture || rc=$?
-          kill "$LOG_PID" 2>/dev/null || true
+          PLATFORM=ios MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" \
+            "$GITHUB_WORKSPACE/$E2E_DIR/run-flows.sh" || rc=$?
+          kill "${log_pids[@]}" 2>/dev/null || true
           exit "$rc"
 
       - name: Enable KVM
@@ -365,7 +351,9 @@ jobs:
           target: google_apis
           arch: x86_64
           force-avd-creation: false
-          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          # read-only so the extra shard emulators can share this AVD: a
+          # writable instance holds a lock the others wait on forever.
+          emulator-options: -read-only -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
           disable-animations: true
           # The action runs each script line in a separate sh -c; the folded
           # scalar (>-) plus && keeps everything in one shell invocation.
@@ -386,9 +374,10 @@ jobs:
             -H "Authorization: Bearer $CLERK_SECRET_KEY" || true
 
       # Test reports record flow env (and typed input) in plaintext;
-      # add-mask only covers step logs, not artifact contents.
+      # add-mask only covers step logs, not artifact contents. Must run whenever
+      # the upload does, or a passing run uploads the password.
       - name: Scrub test credentials from Maestro debug output
-        if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure')
+        if: always() && steps.user.outputs.user_id != ''
         env:
           CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
         run: |
@@ -399,12 +388,13 @@ jobs:
             \( -name '*.html' -o -name '*.json' -o -name '*.log' -o -name '*.txt' -o -name '*.xml' -o -name '*.yaml' \) \
             -exec perl -pi -e 's/\Q$ENV{CLERK_TEST_PASSWORD}\E/[REDACTED]/g' {} +
 
-      - name: Upload Maestro artifacts on e2e failure
-        if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure')
+      - name: Upload Maestro artifacts
+        if: always() && steps.user.outputs.user_id != ''
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: maestro-${{ matrix.platform }}
           path: ${{ runner.temp }}/maestro-debug
+          if-no-files-found: warn
           retention-days: 7
 
       - name: Report e2e outcome

integration/templates/expo-native/App.tsx

@@ -17,6 +17,7 @@ function NativeBuildFixture() {
   const { user } = useUser();
   const [isAuthOpen, setIsAuthOpen] = useState(false);
   const [isProfileOpen, setIsProfileOpen] = useState(false);
+  const [signOutResult, setSignOutResult] = useState<string | null>(null);
 
   if (isProfileOpen) {
     return (
@@ -56,6 +57,7 @@ function NativeBuildFixture() {
 
       <Text testID='auth-state'>{isLoaded ? `signed ${isSignedIn ? 'in' : 'out'}` : 'loading'}</Text>
       {user?.id && <Text testID='user-id'>{user.id}</Text>}
+      {signOutResult && <Text testID='sign-out-result'>{`sign out: ${signOutResult}`}</Text>}
       <Button
         testID='open-auth-view-button'
         title='Open native AuthView'
@@ -73,7 +75,14 @@ function NativeBuildFixture() {
         <Button
           testID='sign-out-button'
           title='Sign out'
-          onPress={() => void signOut()}
+          onPress={() => {
+            setSignOutResult('pending');
+            signOut().then(
+              () => setSignOutResult('ok'),
+              (error: unknown) =>
+                setSignOutResult((error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ')),
+            );
+          }}
         />
       )}
 

integration/tests/expo-native/boot-ios-simulators.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Boots N simulators of one model on the newest installed iOS runtime and
+# configures each for Maestro. `boot` returns as soon as the boots are issued
+# so they overlap the native build; `wait` blocks until every one is usable.
+#
+# Usage: ./boot-ios-simulators.sh boot <count> [model]   # exports MAESTRO_UDID
+#        ./boot-ios-simulators.sh wait                    # reads MAESTRO_UDID
+set -euo pipefail
+
+boot() {
+  local count=$1 model=${2:-iPhone 17 Pro}
+  local runtime device_type udids=() i
+  # The runtime matching the SDK the app was built with, so a newer beta
+  # runtime installed on the runner does not change the OS under test.
+  sdk=$(xcrun --sdk iphonesimulator --show-sdk-version)
+  runtime=$(xcrun simctl list runtimes available -j | jq -r --arg v "$sdk" \
+    '[.runtimes[] | select(.platform == "iOS")] | (map(select(.version == $v)) + .) | first | .identifier')
+  device_type=$(xcrun simctl list devicetypes -j | jq -r --arg m "$model" '.devicetypes[] | select(.name == $m) | .identifier')
+  for i in $(seq 1 "$count"); do
+    udids+=("$(xcrun simctl create "$model $i" "$device_type" "$runtime")")
+    xcrun simctl boot "${udids[$((i - 1))]}"
+  done
+  MAESTRO_UDID=$(IFS=,; echo "${udids[*]}")
+  echo "Booting $count x $model on $runtime: $MAESTRO_UDID"
+  if [ -n "${GITHUB_ENV:-}" ]; then echo "MAESTRO_UDID=$MAESTRO_UDID" >> "$GITHUB_ENV"; fi
+}
+
+wait_ready() {
+  local udid key
+  IFS=, read -r -a udids <<< "${MAESTRO_UDID:?MAESTRO_UDID is required}"
+  for udid in "${udids[@]}"; do
+    # bootstatus blocks with no deadline of its own; a simulator that never
+    # finishes booting would otherwise hold the job until its timeout.
+    xcrun simctl bootstatus "$udid" -b &
+    local pid=$! elapsed=0
+    while kill -0 "$pid" 2>/dev/null; do
+      if [ "$elapsed" -ge 240 ]; then
+        kill "$pid" 2>/dev/null || true
+        wait "$pid" 2>/dev/null || true
+        echo "::error::$udid did not finish booting within ${elapsed}s"
+        xcrun simctl list devices -j | jq -r --arg u "$udid" '.devices[][] | select(.udid == $u) | "\(.name): \(.state)"'
+        return 1
+      fi
+      sleep 5
+      elapsed=$((elapsed + 5))
+    done
+    wait "$pid"
+    # Kill animations + predictive keyboard: animations add latency to every
+    # tap; predictive text hijacks inputText targets.
+    xcrun simctl spawn "$udid" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true
+    xcrun simctl spawn "$udid" defaults write -g ApplePersistenceIgnoreState -bool YES || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.ContinuousPath -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCorrection -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.Prediction -bool NO || true
+    # The one-time keyboard tutorial sheets carry their own Continue button,
+    # which can hijack taps on the AuthView's Continue.
+    for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do
+      xcrun simctl spawn "$udid" defaults write com.apple.keyboard.preferences "$key" -bool YES || true
+    done
+  done
+}
+
+case "${1:-}" in
+  boot) boot "${2:?count is required}" "${3:-}" ;;
+  wait) wait_ready ;;
+  *) echo "usage: $0 boot <count> [model] | wait" >&2; exit 2 ;;
+esac

integration/tests/expo-native/flows/sign-in.yaml

@@ -1,13 +1,22 @@
 # Native AuthView email+password sign-in, asserting the native->JS session
-# sync, then JS-side sign-out asserting the reverse direction.
+# sync; then a restart WITHOUT clearing state, asserting the session is
+# restored from secure-store with no re-auth (bridge + token cache
+# persistence); then JS-side sign-out asserting the reverse direction.
 appId: com.clerk.exponativebuildfixture
-name: Native AuthView sign-in syncs to JS
+name: Native AuthView sign-in syncs to JS and survives a restart
 ---
 - runFlow: subflows/open-app.yaml
 - tapOn:
     id: 'open-auth-view-button'
 - runFlow: subflows/sign-in-email-password.yaml
 - runFlow: subflows/assert-signed-in.yaml
+- stopApp
+- launchApp
+- extendedWaitUntil:
+    visible: 'signed in'
+    timeout: 45000
+- assertVisible:
+    id: 'user-id'
 - tapOn:
     id: 'sign-out-button'
 - runFlow: subflows/assert-signed-out.yaml

integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml

@@ -45,7 +45,7 @@ appId: com.clerk.exponativebuildfixture
                       commands:
                         - tapOn:
                             text: 'Select All'
-            - eraseText: 100
+            - eraseText: 60
             - inputText: ${CLERK_TEST_EMAIL}
       # Wait for the typed value to land before tapping: the tap can race the
       # recomposition that enables the button, and this also catches a mangled
@@ -79,7 +79,6 @@ appId: com.clerk.exponativebuildfixture
     commands:
       - tapOn:
           text: 'Enter your password'
-      - eraseText: 50
       - inputText: ${CLERK_TEST_PASSWORD}
       # The password is masked so its value can't be asserted; settle for the
       # screen going stable so the tap doesn't race the enabling recomposition.

integration/tests/expo-native/run-android-flows.sh

@@ -1,4 +1,11 @@
 #!/usr/bin/env bash
+# Installs the APK on every emulator, starting extra instances of the booted
+# AVD when MAESTRO_SHARDS asks for more than one, then runs the flows across
+# them. Meant to run inside reactivecircus/android-emulator-runner.
+#
+# Usage: ./run-android-flows.sh <apk>
+# Optional env: MAESTRO_SHARDS (default 1), AVD_NAME (default test),
+#               MAESTRO_DEBUG_OUTPUT
 set -euo pipefail
 cd "$(dirname "${BASH_SOURCE[0]}")"
 
@@ -8,21 +15,56 @@ command -v adb >/dev/null 2>&1 || {
   exit 1
 }
 
-adb install -r "$apk_path"
+shards=${MAESTRO_SHARDS:-1}
+devices=(emulator-5554)
+sdk=$(dirname "$(dirname "$(command -v adb)")")
+debug=${MAESTRO_DEBUG_OUTPUT:-${TMPDIR:-/tmp}/clerk-expo-maestro-runner}
+mkdir -p "$debug"
 
-logcat_pid=
+# The action boots one emulator on 5554; extra shards start the same AVD
+# read-only on the next even ports so they share its snapshot.
+for ((i = 1; i < shards; i++)); do
+  port=$((5554 + i * 2))
+  "$sdk/emulator/emulator" -avd "${AVD_NAME:-test}" -read-only -port "$port" \
+    -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot-save \
+    > "$debug/emulator-$port.log" 2>&1 &
+  devices+=("emulator-$port")
+done
+
+# Bounded, because adb wait-for-device on an emulator that died at startup
+# never returns and the job would sit until its timeout with no output.
+wait_for_boot() {
+  local device=$1 elapsed=0
+  until [ "$(adb -s "$device" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r')" = 1 ]; do
+    if [ "$elapsed" -ge 240 ]; then
+      echo "::error::$device did not boot within ${elapsed}s"
+      [ -f "$debug/$device.log" ] && tail -n 40 "$debug/$device.log"
+      return 1
+    fi
+    sleep 5
+    elapsed=$((elapsed + 5))
+  done
+  echo "$device booted"
+}
+for device in "${devices[@]}"; do
+  wait_for_boot "$device"
+  adb -s "$device" install -r "$apk_path"
+done
+
+logcat_pids=()
 stop_logcat() {
-  [ -n "$logcat_pid" ] || return 0
-  kill "$logcat_pid" >/dev/null 2>&1 || true
-  wait "$logcat_pid" 2>/dev/null || true
+  local pid
+  for pid in ${logcat_pids[@]+"${logcat_pids[@]}"}; do
+    kill "$pid" >/dev/null 2>&1 || true
+    wait "$pid" 2>/dev/null || true
+  done
 }
 trap stop_logcat EXIT
 
-if [ -n "${MAESTRO_DEBUG_OUTPUT:-}" ]; then
-  mkdir -p "$MAESTRO_DEBUG_OUTPUT"
-  adb logcat -c || true
-  adb logcat -v threadtime > "$MAESTRO_DEBUG_OUTPUT/android-logcat.log" 2>&1 &
-  logcat_pid=$!
-fi
+for device in "${devices[@]}"; do
+  adb -s "$device" logcat -c || true
+  adb -s "$device" logcat -v threadtime > "$debug/$device-logcat.log" 2>&1 &
+  logcat_pids+=($!)
+done
 
-./run-flows.sh adb shell am force-stop com.clerk.exponativebuildfixture
+PLATFORM=android MAESTRO_UDID=$(IFS=,; echo "${devices[*]}") ./run-flows.sh