Merge pull request #33 from figma/yfang/copy-commits-from-michaelnyu-ci-queue

Merge all commits from michaelnyu/ci-queue

Author: yfang-figma

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  - package-ecosystem: bundler
+    directory: "/ruby/"
+    schedule:
+      interval: weekly
+      time: "14:00"
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    registries: "*"
+    groups:
+      minor-gem-update:
+        update-types:
+          - "minor"
+      patch-gem-update:
+        update-types:
+          - "patch"

.github/workflows/cla.yml

@@ -0,0 +1,23 @@
+# .github/workflows/cla.yml
+name: Contributor License Agreement (CLA)
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+  issue_comment:
+    types: [created]
+
+jobs:
+  cla:
+    runs-on: ubuntu-latest
+    if: |
+      (github.event.issue.pull_request
+        && !github.event.issue.pull_request.merged_at
+        && contains(github.event.comment.body, 'signed')
+      )
+      || (github.event.pull_request && !github.event.pull_request.merged)
+    steps:
+      - uses: Shopify/shopify-cla-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cla-token: ${{ secrets.CLA_TOKEN }}

.github/workflows/tests.yml

@@ -15,9 +15,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
+        ruby:
+          - '2.7'
+          - '3.0'
+          - '3.1'
+          - '3.2'
+          - '3.3.0'
+          - 'truffleruby'
     steps:
       - uses: actions/checkout@v2
+      - name: Install deps
+        run: |
+          sudo apt-get install libsnappy-dev
       - name: Set up Ruby ${{ matrix.ruby }}
         uses: ruby/setup-ruby@v1
         with:

.spin/packages.yml

@@ -0,0 +1 @@
+- libsnappy-dev

README.md

@@ -3,7 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/ci-queue.svg)](https://rubygems.org/gems/ci-queue)
 [![Tests](https://github.com/Shopify/ci-queue/workflows/Tests/badge.svg?branch=master)](https://github.com/Shopify/ci-queue/actions?query=workflow%3ATests)
 
-Distribute tests over many workers using a queue. 
+Distribute tests over many workers using a queue.
 
 ## Why a queue?
 
@@ -24,7 +24,7 @@ Additionally, a separate process can be started to centralize the error reportin
 ## What are requeues?
 
 When working on big test suites, it's not uncommon for a few tests to fail intermittently, either because they are inherently flaky,
-or because they are sensible to other tests modifying some global state (leaky tests).
+or because they are sensitive to other tests modifying some global state (leaky tests).
 
 In this context, it is useful to have mitigation measures so that these intermittent failures don't cause unrelated builds to fail until the root cause is addressed.
 

ruby/isogun.yml renamed to dev.yml

@@ -1,10 +1,8 @@
+---
 # This file is for Shopify employees development environment.
 # If you are an external contributor you don't have to bother with it.
+
 name: ci-queue
 
-vm:
-  ip_address: 192.168.64.245
-  memory: 1G
-  cores: 2
-services:
-   - redis
+up:
+  - snappy

python/ciqueue/_pytest/outcomes.py

@@ -42,6 +42,14 @@ class UnserializableException(Exception):
                      Failed: outcomes.Failed}
 
 
+try:
+    from_exc_info = code.ExceptionInfo.from_exc_info
+except AttributeError:
+    # pytest < 7.4
+    def from_exc_info(tup):
+        return code.ExceptionInfo(tup)
+
+
 def swap_in_serializable(excinfo):
     def pickles(excinfo):
         try:
@@ -52,22 +60,22 @@ def pickles(excinfo):
     if excinfo.type in SERIALIZE_TYPES:
         cls = SERIALIZE_TYPES[excinfo.type]
         tup = (cls, cls(*excinfo.value.args), excinfo.tb)
-        excinfo = code.ExceptionInfo(tup)
+        excinfo = from_exc_info(tup)
     elif not pickles(excinfo):
         tup = (UnserializableException,
                UnserializableException(
                    "Actual Exception thrown on test node was %r" %
                    excinfo.value),
                excinfo.tb)
-        excinfo = code.ExceptionInfo(tup)
+        excinfo = from_exc_info(tup)
     return excinfo
 
 
 def swap_back_original(excinfo):
     if excinfo.type in DESERIALIZE_TYPES:
         tipe = DESERIALIZE_TYPES[excinfo.type]
         tup = (tipe, tipe(*excinfo.value.args), excinfo.tb)
-        return code.ExceptionInfo(tup)
+        return from_exc_info(tup)
     return excinfo
 
 
@@ -84,4 +92,4 @@ def failed(item):
 def skipped_excinfo(item, msg):
     traceback = list(item.error_reports.values())[0]['excinfo'].tb
     tup = (outcomes.Skipped, outcomes.Skipped(msg), traceback)
-    return code.ExceptionInfo(tup)
+    return from_exc_info(tup)

python/ciqueue/_pytest/test_queue.py

@@ -52,6 +52,8 @@ def parse_redis_args(spec):
         result['socket_connect_timeout'] = int(query['socket_connect_timeout'][0])
     if 'retry_on_timeout' in query:
         result['retry_on_timeout'] = bool(util.strtobool(query['retry_on_timeout'][0] or 'false'))
+    if spec.scheme == "rediss":
+        result['ssl'] = True
 
     return result
 
@@ -62,7 +64,7 @@ def build_queue(queue_url, tests_index=None):
         return ciqueue.Static(spec.path.split(':'))
     elif spec.scheme == 'file':
         return ciqueue.File(spec.path)
-    elif spec.scheme == 'redis':
+    elif spec.scheme == 'redis' or spec.scheme == 'rediss':
         redis_args = parse_redis_args(spec)
         redis_client = redis.StrictRedis(**redis_args)
 

python/setup.py

@@ -37,7 +37,7 @@ def get_lua_scripts():
     packages=['ciqueue', 'ciqueue._pytest'],
     install_requires=[
         'dill>=0.2.7',
-        'pytest>=2.7,<7',
+        'pytest>=2.7',
         'redis>=2.10.5',
         'tblib>=1.3.2',
         'uritools>=2.0.0',

python/tests/test_pytest.py

@@ -1,14 +1,21 @@
 import os
+import re
 import subprocess
 import redis
+import pytest
 
 # pylint: disable=no-self-use
 
 
+@pytest.fixture(autouse=True)
+def change_test_dir(request, monkeypatch):
+    monkeypatch.chdir(request.fspath.dirname + "/..")
+
+
 def expected_messages(output):
     assert '= 4 failed, 2 passed, 1 skipped, 1 xpassed, 6 errors in' in output, output
-    assert ('integrations/pytest/test_all.py:27: skipping test message' in output
-            or 'integrations/pytest/test_all.py:28: skipping test message' in output), output
+    assert re.search(r':\d+: skipping test message', output) is not None, \
+        "did not find 'skipping test message' in output"
 
 
 def check_output(cmd):