chore(CTS): add retry on tests (#672)

* chore(cts): fix api keys test

* chore(CTS): fix flakiness

Author: chloelbn

src/SearchClient.php

@@ -155,7 +155,7 @@ public function isAlive($requestOptions = array())
 
     public function multipleQueries($queries, $requestOptions = array())
     {
-        $queries = array_map(function($query) {
+        $queries = array_map(function ($query) {
             $query['params'] = isset($query['params']) ?
                 Helpers::serializeQueryParameters($query['params']) :
                 Helpers::serializeQueryParameters(array());

tests/Integration/AnalyticsClientTest.php

@@ -61,7 +61,10 @@ public function testAbTesting()
             $cpt++;
         } while (false);
 
-        $response = $analyticsClient->addABTest($abTest);
+        $response = TestHelper::retry(function () use ($analyticsClient, $abTest) {
+            return $analyticsClient->addABTest($abTest);
+        }, 0.1, 40);
+
         $abTestId = $response['abTestID'];
         $index->waitTask($response['taskID']);
 
@@ -160,7 +163,10 @@ public function testAaTesting()
             $cpt++;
         } while (false);
 
-        $response = $analyticsClient->addABTest($aaTest);
+        $response = TestHelper::retry(function () use ($analyticsClient, $aaTest) {
+            return $analyticsClient->addABTest($aaTest);
+        }, 0.1, 40);
+
         $aaTestId = $response['abTestID'];
         TestHelper::getClient()->waitTask($this->indexes['aa_testing'], $response['taskID']);
 

tests/Integration/BaseTest.php

@@ -15,6 +15,7 @@ public static function setUpBeforeClass()
             TestHelper::checkEnvironmentVariables();
         } catch (\Exception $e) {
             echo $e->getMessage()."\n";
+
             return;
         }
 

tests/Integration/SearchClientTest.php

@@ -2,7 +2,6 @@
 
 namespace Algolia\AlgoliaSearch\Tests\Integration;
 
-use Algolia\AlgoliaSearch\Exceptions\NotFoundException;
 use Algolia\AlgoliaSearch\Response\MultiResponse;
 use Algolia\AlgoliaSearch\SearchClient;
 use Algolia\AlgoliaSearch\SearchIndex;
@@ -271,29 +270,13 @@ public function testApiKeys()
 
         TestHelper::getClient()->updateApiKey($res['key'], $newParams)->wait();
 
-        $retry = 1;
-        $time = 100000;
-        $maxRetries = 100;
-        do {
-            if ($retry >= $maxRetries) {
-                break;
-            }
+        $updatedApiKey = TestHelper::retry(function () use ($res) {
+            return TestHelper::getClient()->getApiKey($res['key']);
+        });
 
-            try {
-                $updatedApiKey = TestHelper::getClient()->getApiKey($res['key']);
-
-                if ($updatedApiKey['maxHitsPerQuery'] !== $apiKey['maxHitsPerQuery']) {
-                    $this->assertEquals(42, $updatedApiKey['maxHitsPerQuery']);
-                    break;
-                }
-            } catch (NotFoundException $e) {
-                // Try again
-            }
-
-            $retry++;
-            $factor = ceil($retry / 10);
-            usleep($factor * $time); // 0.1 second
-        } while (true);
+        if ($updatedApiKey['maxHitsPerQuery'] !== $apiKey['maxHitsPerQuery']) {
+            $this->assertEquals(42, $updatedApiKey['maxHitsPerQuery']);
+        }
 
         TestHelper::getClient()->deleteApiKey($res['key'])->wait();
 
@@ -303,9 +286,14 @@ public function testApiKeys()
             $this->assertInstanceOf('Algolia\AlgoliaSearch\Exceptions\NotFoundException', $e);
         }
 
-        TestHelper::getClient()->restoreApiKey($res['key'])->wait();
+        TestHelper::retry(function () use ($res) {
+            TestHelper::getClient()->restoreApiKey($res['key'])->wait();
+        });
+
+        $restoredApiKey = TestHelper::retry(function () use ($res) {
+            return TestHelper::getClient()->getApiKey($res['key']);
+        });
 
-        $restoredApiKey = TestHelper::getClient()->getApiKey($res['key']);
         $this->assertEquals($acl, $restoredApiKey['acl']);
         $this->assertEquals($params['description'], $restoredApiKey['description']);
 

tests/Integration/SecuredApiKeysTest.php

@@ -52,7 +52,10 @@ public function testSecuredApiKeys()
         /** @var SearchIndex $securedIndexDev */
         $securedIndexDev = $securedClient->initIndex($this->indexes['secured_api_keys_dev']);
 
-        $res = $securedIndex->search('');
+        $res = TestHelper::retry(function () use ($securedIndex) {
+            return $securedIndex->search('');
+        });
+
         $this->assertCount(1, $res['hits']);
 
         try {

tests/TestHelper.php

@@ -3,6 +3,7 @@
 namespace Algolia\AlgoliaSearch\Tests;
 
 use Algolia\AlgoliaSearch\Config\SearchConfig;
+use Algolia\AlgoliaSearch\Exceptions\NotFoundException;
 use Algolia\AlgoliaSearch\SearchClient;
 use Faker\Factory;
 
@@ -107,4 +108,27 @@ public static function formatRule($rule)
 
         return $rule;
     }
+
+    /**
+     * @param mixed $function   function on which to retry
+     * @param float $delay      delay for the retry, in seconds
+     * @param int   $maxRetries maximum of retries
+     *
+     * @return mixed
+     *
+     * @throws \Exception
+     */
+    public static function retry($function, $delay = 0.1, $maxRetries = 30)
+    {
+        for ($i = 1; $i < $maxRetries; $i++) {
+            try {
+                return $function();
+            } catch (NotFoundException $e) {
+                usleep($delay * 1000000); // 0.1 second
+                continue;
+            }
+        }
+
+        throw new \Exception('reached the maximum number of retries');
+    }
 }