Merge pull request #8 from crowdsecurity/feature/bouncer-prunable

cache is now prunable

Author: mobula9

.github/workflows/tests.yml

@@ -45,7 +45,6 @@ jobs:
             -   name: Add a bouncer to run phpunit tests
                 run: docker exec crowdsec cscli bouncers add bouncer-php-library -o raw > .bouncer-key
 
-            # TODO P2 Move values to env vars
             -   name: Add a machine to pilot crowdsec state
                 run: docker exec crowdsec cscli machines add PhpUnitTestMachine --password PhpUnitTestMachinePassword
 

src/ApiCache.php

@@ -5,6 +5,7 @@
 namespace CrowdSecBouncer;
 
 use Symfony\Component\Cache\Adapter\AbstractAdapter;
+use Symfony\Component\Cache\PruneableInterface;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -98,7 +99,7 @@ private function addRemediationToCacheItem(string $ip, string $type, int $expira
         // Build the item lifetime in cache and sort remediations by priority
         $maxLifetime = max(array_column($remediations, 1));
         $prioritizedRemediations = Remediation::sortRemediationByPriority($remediations);
-        
+
         $item->set($prioritizedRemediations);
         $item->expiresAfter($maxLifetime);
 
@@ -205,7 +206,7 @@ private function formatRemediationFromDecision(?array $decision): array
         }
 
         return [
-            $decision['type'], // ex: captcha
+            $decision['type'], // ex: ban, captcha
             time() + self::parseDurationToSeconds($decision['duration']), // expiration timestamp
             $decision['id'],
         ];
@@ -222,12 +223,6 @@ private function defferUpdateCacheConfig(array $config): void
 
     /**
      * Update the cached remediations from these new decisions.
-
-     * TODO P2 WRITE TESTS
-     * 0 decisions
-     * 3 known remediation type
-     * 3 decisions but 1 unknown remediation type
-     * 3 unknown remediation type
      */
     private function saveRemediations(array $decisions): bool
     {
@@ -278,6 +273,12 @@ private function saveRemediationsForIp(array $decisions, string $ip): void
     {
         if (\count($decisions)) {
             foreach ($decisions as $decision) {
+                if (!in_array($decision['type'], Constants::ORDERED_REMEDIATIONS)) {
+                    $highestRemediationLevel = Constants::ORDERED_REMEDIATIONS[0];
+                    // TODO P1 test the case of unknown remediation type
+                    $this->logger->warning("The remediation type " . $decision['type'] . " is unknown by this CrowdSec Bouncer version. Fallback to highest remedition level: " . $highestRemediationLevel);
+                    $decision['type'] = $highestRemediationLevel;
+                }
                 $remediation = $this->formatRemediationFromDecision($decision);
                 $this->addRemediationToCacheItem($ip, $remediation[0], $remediation[1], $remediation[2]);
             }
@@ -288,12 +289,17 @@ private function saveRemediationsForIp(array $decisions, string $ip): void
         $this->adapter->commit();
     }
 
+    public function clear(): bool
+    {
+        return $this->adapter->clear();
+    }
+
     /**
      * Used in stream mode only.
      * Warm the cache up.
      * Used when the stream mode has just been activated.
      *
-     * TODO P2 test for overlapping decisions strategy (max expires, remediation ordered by priorities)
+     * TODO P2 test for overlapping decisions strategy (ex: max expires)
      */
     public function warmUp(): void
     {
@@ -302,7 +308,7 @@ public function warmUp(): void
         $decisionsDiff = $this->apiClient->getStreamedDecisions($startup);
         $newDecisions = $decisionsDiff['new'];
 
-        $this->adapter->clear();
+        $this->clear();
 
         if ($newDecisions) {
             $this->warmedUp = $this->saveRemediations($newDecisions);
@@ -366,7 +372,8 @@ private function miss(string $ip): string
     private function hit(string $ip): string
     {
         $remediations = $this->adapter->getItem($ip)->get();
-        // TODO P2 control before if date is not expired and if true, update cache item.
+        // TODO P1 foreach $remediations, control if exp date is not expired.
+        // If true, update cache item by removing this expired remediation.
 
         // We apply array values first because keys are ids.
         $firstRemediation = array_values($remediations)[0];
@@ -398,4 +405,19 @@ public function get(string $ip): string
             return $this->miss($ip);
         }
     }
+
+    public function prune(): bool
+    {
+        $isPrunable = ($this->adapter instanceof PruneableInterface);
+        if (!$isPrunable) {
+            throw new BouncerException("Cache Adapter" . get_class($this->adapter) . " is not prunable.");
+        }
+        /** @var PruneableInterface */
+        $adapter = $this->adapter;
+        $pruned = $adapter->prune();
+        $this->logger->info('Cached adapter pruned');
+
+        // TODO P3 Prune remediation inside cache items.
+        return $pruned;
+    }
 }

src/ApiClient.php

@@ -51,7 +51,7 @@ public function configure(string $baseUri, int $timeout, string $userAgent, stri
      */
     public function getFilteredDecisions(array $filter): array
     {
-        // TODO P2 keep results filtered for scope=ip or scope=range (we can't do anything with other scopes)
+        // TODO P1 keep results filtered for scope=ip or scope=range (we can't do anything with other scopes)
         $decisions = $this->restClient->request('/v1/decisions', $filter);
         $decisions = $decisions ?: [];
 
@@ -64,7 +64,7 @@ public function getFilteredDecisions(array $filter): array
      */
     public function getStreamedDecisions(bool $startup = false): array
     {
-        // TODO P2 keep results filtered for scope=ip or scope=range (we can't do anything with other scopes)
+        // TODO P1 keep results filtered for scope=ip or scope=range (we can't do anything with other scopes)
         /** @var array */
         $decisionsDiff = $this->restClient->request('/v1/decisions/stream', $startup ? ['startup' => 'true'] : null);
 

src/Bouncer.php

@@ -129,6 +129,24 @@ public function refreshBlocklistCache(): void
         $this->apiCache->pullUpdates();
     }
 
+    /**
+     * This method clear the full data in cache.
+     */
+    public function clearCache(): bool
+    {
+        return $this->apiCache->clear();
+    }
+
+    /**
+     * This method prune the cache: it removes all the expired cache items.
+     */
+    public function pruneCache(): bool
+    {
+        return $this->apiCache->clear();
+    }
+
+    
+
     /**
      * Browse the remediations cache.
      */

src/Constants.php

@@ -15,16 +15,16 @@
 class Constants
 {
     /** @var string The URL of the CrowdSec Central API */
-    public const CAPI_URL = 'https://api.crowdsec.net/v2/'; // TODO P2 get the correct one
+    public const CAPI_URL = 'https://api.crowdsec.net/v2/';
 
     /** @var string The user agent used to send request to LAPI or CAPI */
-    public const BASE_USER_AGENT = 'CrowdSec PHP Library/1.0.0'; // TODO P3 get the correct version
+    public const BASE_USER_AGENT = 'PHP CrowdSec Bouncer/1.0.0'; // TODO P3 get the correct version
 
     /** @var int The timeout when calling LAPI or CAPI */
-    public const API_TIMEOUT = 1; // TODO P2 get the correct one
+    public const API_TIMEOUT = 1;
 
     /** @var int The duration we keep a clean IP in cache 600s = 10m */
-    public const CACHE_EXPIRATION_FOR_CLEAN_IP = 600; // TODO P2 get the correct one
+    public const CACHE_EXPIRATION_FOR_CLEAN_IP = 600; // TODO P1 get the correct one
 
     /** @var string The ban remediation */
     public const REMEDIATION_BAN = 'ban';

tests/IpVerificationTest.php

@@ -11,18 +11,18 @@
 use Symfony\Component\Cache\Adapter\AbstractAdapter;
 use PHPUnit\Framework\MockObject\MockObject;
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Cache\PruneableInterface;
 
 define("HOST_IS_UP", true);
 define("HOST_IS_DOWN", false);
 
 /*
 TODO P2 Instanciate all the configuration
-TODO P2 testThrowErrorWhenMissAndApiIsNotReachable()
+TODO P2 testThrowErrorWhenMissAndApiIsNotReachable() https://stackoverflow.com/questions/5683592/phpunit-assert-that-an-exception-was-thrown
 TODO P2 testThrowErrorWhenMissAndApiTimeout()
-TODO P2 testCanVerifyCaptchableIp()
 TODO P2 testCanHandleCacheSaturation()
-TODO P2 testCanNotUseCapiInLiveMode()
-TODO P2 testCanVerifyIpInStreamModeWithCacheSystemBeforeWarmingTheCacheUp() https://stackoverflow.com/questions/5683592/phpunit-assert-that-an-exception-was-thrown
+TODO P3 testCanNotUseCapiInLiveMode()
+TODO P2 testCanVerifyIpInStreamModeWithCacheSystemBeforeWarmingTheCacheUp()
 */
 
 final class IpVerificationTest extends TestCase
@@ -65,6 +65,7 @@ public function testCanVerifyIpInLiveModeWithoutCacheSystem(): void
         $remediation = $bouncer->getRemediationForIp($badIp);
         $this->assertEquals($remediation, 'ban');
     }*/
+
     /**
      * @group integration
      * @covers Bouncer
@@ -122,8 +123,13 @@ public function testCanVerifyIpInLiveModeWithCacheSystem(AbstractAdapter $cacheA
         $cleanRemediation2ndCall = $bouncer->getRemediationForIp($cleanIp);
         $this->assertEquals('bypass', $cleanRemediation2ndCall);
 
+        // Prune cache
+        if ($cacheAdapter instanceof PruneableInterface) {
+            $this->assertTrue($bouncer->pruneCache(), 'The cache should be prunable');
+        }
+
         // Clear cache
-        $cacheAdapter->clear();
+        $this->assertTrue($bouncer->clearCache(), 'The cache should be clearable');
 
         // Call one more time (should miss as the cache has been cleared)
 

tests/Template403Test.php

@@ -9,11 +9,11 @@ final class Template403Test extends TestCase
 {
     /**
      * @group integration
-     * @covers \CrowdSecBouncer\Bouncer
+     * @covers Bouncer
      */
     public function testCanGetDefault403Template(): void
     {
-        // TODO P2 update these tests
+        // TODO P1 update these tests
         //$bouncer = new Bouncer();
         //$bouncer->configure($config, $cacheAdapter);
         //$this->assertIsString($bouncer->getdefault403Template());