Replace the FTL test script with FTL GHA (#1060)

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -988,9 +988,10 @@ jobs:
           timeout_minutes: 5
           max_attempts: 3
           command: scripts/gha/install_test_workflow_prereqs.sh -p Android -t true
-      - id: get-device-type
+      - id: device-info
         run: |
-          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.android_device }} )"
+          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_device_type)"
+          echo "::set-output name=device::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_ftl_device)"
       - name: Set up Node (16)
         uses: actions/setup-node@v2
         with:
@@ -1001,15 +1002,15 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Setup Firestore Emulator
-        if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
         uses: nick-invision/retry@v2
         with:
           shell: bash
           timeout_minutes: 5
           max_attempts: 3
           command: npm install -g firebase-tools     
       - name: Start Firestore Emulator
-        if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
         run: |
           firebase emulators:start --only firestore --project demo-example &
       - name: Setup java 8 for test_simulator.py
@@ -1018,27 +1019,31 @@ jobs:
           distribution: 'temurin'
           java-version: '8'
       - name: Run Android integration tests on Emulator locally
-        timeout-minutes: 120
-        if: steps.get-device-type.outputs.device_type == 'virtual'
+        timeout-minutes: 150
+        if: steps.device-info.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
             --test_type "${{ matrix.test_type }}" \
             --android_device "${{ matrix.android_device }}" \
             --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}" \
             --ci
-      - name: Install Cloud SDK
-        if: steps.get-device-type.outputs.device_type == 'real'
-        uses: google-github-actions/setup-gcloud@v0
-      - name: Run Android integration tests on Real Device via FTL
-        timeout-minutes: 60
-        if: steps.get-device-type.outputs.device_type == 'real'
+      - id: ftl_test
+        if: steps.device-info.outputs.device_type == 'real'
+        uses: FirebaseExtended/github-actions/[email protected]
+        timeout-minutes: 90
+        with:
+          credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
+          testapp_dir: testapps
+          test_type: "game-loop"
+          test_devices: ${{ steps.device-info.outputs.device }}
+          max_attempts: 3
+          validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
+      - name: Read FTL Test Result
+        if: steps.device-info.outputs.device_type == 'real'
+        shell: bash
         run: |
-          python scripts/gha/test_lab.py --testapp_dir testapps \
-            --android_device "${{ matrix.android_device }}" \
-            --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}" \
-            --code_platform cpp \
-            --key_file_encrypted scripts/gha-encrypted/gcs_key_file.json.gpg \
-            --key_file_passphrase "${{ secrets.TEST_SECRET }}"
+          python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
+            --output_path testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log
       - name: Prepare results summary artifact
         if: ${{ !cancelled() }}
         shell: bash
@@ -1055,14 +1060,14 @@ jobs:
           path: testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}*
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Upload Android test video artifact
-        if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
         uses: actions/upload-artifact@v3
         with:
           name: mobile-simulator-test-video-artifact
           path: testapps/video-*-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.mp4
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Upload Android test logcat artifact
-        if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
         uses: actions/upload-artifact@v3
         with:
           name: mobile-simulator-test-logcat-artifact
@@ -1130,9 +1135,10 @@ jobs:
           timeout_minutes: 3
           max_attempts: 3
           command: scripts/gha/install_test_workflow_prereqs.sh -p iOS -t true
-      - id: get-device-type
+      - id: device-info
         run: |
-          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.ios_device }} )"
+          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_device_type)"
+          echo "::set-output name=device::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_ftl_device)"
       - name: Set up Node (16)
         uses: actions/setup-node@v2
         with:
@@ -1143,40 +1149,43 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Setup Firestore Emulator
-        if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
         uses: nick-invision/retry@v2
         with:
           shell: bash
           timeout_minutes: 5
           max_attempts: 3
           command: npm install -g firebase-tools     
       - name: Start Firestore Emulator
-        if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
         run: |
           firebase emulators:start --only firestore --project demo-example &
       - name: Run iOS integration tests on Simulator locally
-        timeout-minutes: 120
-        if: steps.get-device-type.outputs.device_type == 'virtual'
+        timeout-minutes: 150
+        if: steps.device-info.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
             --test_type "${{ matrix.test_type }}" \
             --ios_device "${{ matrix.ios_device }}" \
             --logfile_name "ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}" \
             --ci
-      - name: Install Cloud SDK
-        if: steps.get-device-type.outputs.device_type == 'real'
-        uses: google-github-actions/setup-gcloud@v0
-      - name: Run iOS integration tests on Real Device via FTL
-        # max 3 retry and 10m timeout for each testapp, plus other steps
-        timeout-minutes: 60
-        if: steps.get-device-type.outputs.device_type == 'real'
+      - id: ftl_test
+        if: steps.device-info.outputs.device_type == 'real'
+        uses: FirebaseExtended/github-actions/[email protected]
+        timeout-minutes: 90
+        with:
+          credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
+          testapp_dir: testapps
+          test_type: "game-loop"
+          test_devices: ${{ steps.device-info.outputs.device }}
+          max_attempts: 3
+          validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
+      - name: Read FTL Test Result
+        if: steps.device-info.outputs.device_type == 'real'
+        shell: bash
         run: |
-          python scripts/gha/test_lab.py --testapp_dir testapps \
-            --ios_device "${{ matrix.ios_device }}" \
-            --logfile_name "ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}" \
-            --code_platform cpp \
-            --key_file_encrypted scripts/gha-encrypted/gcs_key_file.json.gpg \
-            --key_file_passphrase "${{ secrets.TEST_SECRET }}"
+          python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
+            --output_path testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log
       - name: Prepare results summary artifact
         if: ${{ !cancelled() }}
         shell: bash
@@ -1193,7 +1202,7 @@ jobs:
           path: testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}*
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Upload iOS test video artifact
-        if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
         uses: actions/upload-artifact@v3
         with:
           name: mobile-simulator-test-video-artifact

scripts/gha/integration_testing/ftl_gha_validator.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# Copyright 2022 Google
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import subprocess
+import shutil
+import re
+
+GSUTIL = shutil.which("gsutil")
+
+def validate(test_summary):
+  if not test_summary:
+    logging.error("Nothing to be validate! Please provide test_summary")
+    return False
+
+  app_path = test_summary.get("testapp_path")
+  return_code = test_summary.get("return_code")
+  logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
+  if return_code == 0:
+    gcs_dir = test_summary.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
+    logging.info("gcs_dir: %s" % gcs_dir)
+    logs = _get_testapp_log_text_from_gcs(gcs_dir)
+    logging.info("Test result: %s", logs)
+    return _validate_logs(logs)
+  else:
+    logging.error("Test failed: %s", app_path)
+    return False
+
+
+def _get_testapp_log_text_from_gcs(gcs_path):
+  """Gets the testapp log text generated by game loops."""
+  try:
+    gcs_contents = _gcs_list_dir(gcs_path)
+  except subprocess.CalledProcessError as e:
+    logging.error("Unexpected error searching GCS logs:\n%s", e)
+    return None
+  # The path to the testapp log depends on the platform, device, and scenario
+  # being tested. Search for a json file with 'results' in the name to avoid
+  # hard-coding too many assumptions about the path. The testapp log should be
+  # the only json, but 'results' adds some redundancy in case this changes.
+  matching_gcs_logs = [
+    line for line in gcs_contents
+    if line.endswith(".json") and "results" in line.lower()
+  ]
+  if not matching_gcs_logs:
+    logging.error("Failed to find results log on GCS.")
+    return None
+  # This assumes testapps only have one scenario. Could change in the future.
+  if len(matching_gcs_logs) > 1:
+    logging.warning("Multiple scenario logs found.")
+  gcs_log = matching_gcs_logs[0]
+  try:
+    logging.info("Found results log: %s", gcs_log)
+    log_text = _gcs_read_file(gcs_log)
+    if not log_text:
+      logging.warning("Testapp log is empty. App may have crashed.")
+    return log_text
+  except subprocess.CalledProcessError as e:
+    logging.error("Unexpected error reading GCS log:\n%s", e)
+    return None
+
+
+def _gcs_list_dir(gcs_path):
+  """Recursively returns a list of contents for a directory on GCS."""
+  args = [GSUTIL, "ls", "-r", gcs_path]
+  logging.info("Listing GCS contents: %s", " ".join(args))
+  result = subprocess.run(args=args, capture_output=True, text=True, check=True)
+  return result.stdout.splitlines()
+
+
+def _gcs_read_file(gcs_path):
+  """Extracts the contents of a file on GCS."""
+  args = [GSUTIL, "cat", gcs_path]
+  logging.info("Reading GCS file: %s", " ".join(args))
+  result = subprocess.run(args=args, capture_output=True, text=True, check=True)
+  return result.stdout
+
+
+def _validate_logs(log_text):
+  if not log_text:
+    return False
+ 
+  # The gtest runner dumps a useful summary of tests after the tear down.
+  end_marker = "Global test environment tear-down"
+  complete = end_marker in log_text
+
+  if complete:
+    # rpartition splits a string into three components around the final
+    # occurrence of the end marker, returning a triplet (before, marker, after)
+    result_summary = log_text.rpartition(end_marker)[2].lstrip()
+    failures = re.search(r"\[  FAILED  \] (?P<count>[0-9]+) test", result_summary)
+    logging.info("_validate_logs failures: %s", failures)
+    if failures:
+      return False
+    else:
+      return True
+  else:
+    return False

scripts/gha/print_matrix_configuration.py

@@ -161,17 +161,20 @@
   "tvos": ["os", "xcode_version", "tvos_device"]
 }
 
+# Check currently supported models and versions with the following commands:
+#   gcloud firebase test android models list
+#   gcloud firebase test ios models list
 TEST_DEVICES = {
-  "android_min": {"type": "real", "model":"Nexus10", "version":"19"},
-  "android_target": {"type": "real", "model":"gts4lltevzw", "version":"28"},
-  "android_latest": {"type": "real", "model":"redfin", "version":"30"},
+  "android_min": {"type": "real", "device": "model=Nexus10,version=19"},
+  "android_target": {"type": "real", "device": "model=gts4lltevzw,version=28"},
+  "android_latest": {"type": "real", "device": "model=oriole,version=33"},
   "emulator_min": {"type": "virtual", "image":"system-images;android-18;google_apis;x86"},
   "emulator_target": {"type": "virtual", "image":"system-images;android-30;google_apis;x86_64"},
   "emulator_latest": {"type": "virtual", "image":"system-images;android-32;google_apis;x86_64"},
   "emulator_32bit": {"type": "virtual", "image":"system-images;android-30;google_apis;x86"},
-  "ios_min": {"type": "real", "model":"iphone8", "version":"12.4"},
-  "ios_target": {"type": "real", "model":"iphone8", "version":"13.6"},
-  "ios_latest": {"type": "real", "model":"iphone11pro", "version":"14.7"},
+  "ios_min": {"type": "real", "device": "model=iphonexr,version=13.2"},
+  "ios_target": {"type": "real", "device": "model=iphone8,version=13.6"},
+  "ios_latest": {"type": "real", "device": "model=iphone11pro,version=14.7"},
   "simulator_min": {"type": "virtual", "name":"iPhone 8", "version":"13.7"},
   "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"14.5"},
   "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"15.2"},
@@ -356,9 +359,12 @@ def main():
     print_value(args.override)
     return
 
-  if args.device:
+  if args.get_device_type:
     print(TEST_DEVICES.get(args.parm_key).get("type"))
     return 
+  if args.get_ftl_device:
+    print(TEST_DEVICES.get(args.parm_key).get("device"))
+    return 
 
   if args.expanded:
     test_matrix = EXPANDED_KEY
@@ -383,7 +389,8 @@ def parse_cmdline_args():
   parser.add_argument('-k', '--parm_key', required=True, help='Print the value of specified key from matrix or config maps.')
   parser.add_argument('-a', '--auto_diff', metavar='BRANCH', help='Compare with specified base branch to automatically set matrix options')
   parser.add_argument('-o', '--override', help='Override existing value with provided value')
-  parser.add_argument('-d', '--device', action='store_true', help='Get the device type, used with -k $device')
+  parser.add_argument('-get_device_type', action='store_true', help='Get the device type, used with -k $device')
+  parser.add_argument('-get_ftl_device', action='store_true', help='Get the ftl test device, used with -k $device')
   parser.add_argument('-t', '--device_type', default=['real', 'virtual'], help='Test on which type of mobile devices')
   parser.add_argument('--apis', default=PARAMETERS["integration_tests"]["config"]["apis"], 
                       help='Exclude platform based on apis. Certain platform does not support all apis. e.g. tvOS does not support messaging')