Skip to content

Commit 5bca077

Browse files
shiftedrealityshiftedreality
authored
MCLOUD-6023: Investigate Redis scan keys eviction (#31)
1 parent 74dde3d commit 5bca077

File tree

6 files changed

+256
-0
lines changed

6 files changed

+256
-0
lines changed

‎Console/Command/CacheEvict.php

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\CloudComponents\Console\Command;
9+
10+
use Magento\CloudComponents\Model\Cache\Evictor;
11+
use Symfony\Component\Console\Command\Command;
12+
use Symfony\Component\Console\Input\InputInterface;
13+
use Symfony\Component\Console\Output\OutputInterface;
14+
15+
/**
16+
* Performs force key eviction with a "scan" command.
17+
*/
18+
class CacheEvict extends Command
19+
{
20+
/**
21+
* @var Evictor
22+
*/
23+
private $evictor;
24+
25+
/**
26+
* @param Evictor $evictor
27+
*/
28+
public function __construct(Evictor $evictor)
29+
{
30+
$this->evictor = $evictor;
31+
32+
parent::__construct('cache:evict');
33+
}
34+
35+
/**
36+
* @inheritDoc
37+
*/
38+
protected function configure()
39+
{
40+
$this->setDescription('Evicts unused keys by performing scan command');
41+
}
42+
43+
/**
44+
* @inheritDoc
45+
*/
46+
public function execute(InputInterface $input, OutputInterface $output)
47+
{
48+
$output->writeln('Begin scanning of cache keys');
49+
50+
$count = $this->evictor->evict();
51+
52+
$output->writeln(sprintf(
53+
'Total scanned keys: %s',
54+
$count
55+
));
56+
}
57+
}

‎Cron/Evict.php

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\CloudComponents\Cron;
9+
10+
use Magento\CloudComponents\Model\Cache\Evictor;
11+
use Magento\Framework\App\DeploymentConfig;
12+
use Psr\Log\LoggerInterface;
13+
14+
/**
15+
* The cron cprocess to evict keys.
16+
*/
17+
class Evict
18+
{
19+
/**
20+
* @var Evictor
21+
*/
22+
private $evictor;
23+
24+
/**
25+
* @var DeploymentConfig
26+
*/
27+
private $deploymentConfig;
28+
29+
/**
30+
* @var LoggerInterface
31+
*/
32+
private $logger;
33+
34+
/**
35+
* @param Evictor $evictor
36+
* @param DeploymentConfig $deploymentConfig
37+
* @param LoggerInterface $logger
38+
*/
39+
public function __construct(Evictor $evictor, DeploymentConfig $deploymentConfig, LoggerInterface $logger)
40+
{
41+
$this->evictor = $evictor;
42+
$this->deploymentConfig = $deploymentConfig;
43+
$this->logger = $logger;
44+
}
45+
46+
/**
47+
* Perform keys eviction.
48+
*/
49+
public function execute()
50+
{
51+
if (!$this->deploymentConfig->get(Evictor::CONFIG_PATH_ENABLED)) {
52+
$this->logger->info('Keys eviction is disabled');
53+
54+
return;
55+
}
56+
57+
$this->evictor->evict();
58+
}
59+
}

‎Model/Cache/Evictor.php

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\CloudComponents\Model\Cache;
9+
10+
use Magento\Framework\App\Cache\Type\FrontendPool;
11+
use Magento\Framework\App\DeploymentConfig;
12+
use Psr\Log\LoggerInterface;
13+
use Credis_Client as Client;
14+
use Cm_Cache_Backend_Redis as Backend;
15+
16+
/**
17+
* Performs force key eviction with a "scan" command.
18+
*/
19+
class Evictor
20+
{
21+
const DEFAULT_EVICTION_LIMIT = 10000;
22+
const DEFAULT_SLEEP_TIMEOUT = 20000;
23+
const CONFIG_PATH_ENABLED = 'cache_evict/enabled';
24+
const CONFIG_PATH_LIMIT = 'cache_evict/limit';
25+
26+
/**
27+
* @var DeploymentConfig
28+
*/
29+
private $deploymentConfig;
30+
31+
/**
32+
* @var LoggerInterface
33+
*/
34+
private $logger;
35+
36+
/**
37+
* @param DeploymentConfig $deploymentConfig
38+
* @param LoggerInterface $logger
39+
*/
40+
public function __construct(DeploymentConfig $deploymentConfig, LoggerInterface $logger)
41+
{
42+
$this->deploymentConfig = $deploymentConfig;
43+
$this->logger = $logger;
44+
}
45+
46+
/**
47+
* Evicts all keys using iterator.
48+
*
49+
* @return int
50+
*/
51+
public function evict(): int
52+
{
53+
$options = $this->deploymentConfig->getConfigData(FrontendPool::KEY_CACHE)[FrontendPool::KEY_FRONTEND_CACHE]
54+
?? [];
55+
$evictedKeys = 0;
56+
57+
foreach ($options as $name => $cacheConfig) {
58+
$this->logger->info(sprintf(
59+
'Scanning keys for "%s" database',
60+
$name
61+
));
62+
63+
if (!isset(
64+
$cacheConfig['backend_options']['server'],
65+
$cacheConfig['backend_options']['port'],
66+
$cacheConfig['backend_options']['database']
67+
)) {
68+
$this->logger->debug(sprintf(
69+
'Cache config for database "%s" config is not valid',
70+
$name
71+
));
72+
73+
continue;
74+
}
75+
76+
$dbKeys = $this->run(
77+
$cacheConfig['backend_options']['server'],
78+
$cacheConfig['backend_options']['port'],
79+
$cacheConfig['backend_options']['database']
80+
);
81+
$evictedKeys += $dbKeys;
82+
83+
$this->logger->info(sprintf('Keys scanned: %s', $dbKeys));
84+
}
85+
86+
return $evictedKeys;
87+
}
88+
89+
/**
90+
* @param string $host
91+
* @param int $port
92+
* @param int $db
93+
* @return int
94+
*/
95+
private function run(string $host, int $port, int $db): int
96+
{
97+
$client = new Client($host, $port, null, '', $db);
98+
$evictedKeys = 0;
99+
100+
do {
101+
$keys = $client->scan(
102+
$iterator,
103+
Backend::PREFIX_KEY . '*',
104+
(int)$this->deploymentConfig->get(self::CONFIG_PATH_LIMIT, self::DEFAULT_EVICTION_LIMIT)
105+
);
106+
107+
if ($keys === false) {
108+
$this->logger->debug('Reached end');
109+
} else {
110+
$keysCount = count($keys);
111+
$evictedKeys += $keysCount;
112+
}
113+
114+
/* Give Redis some time to handle other requests */
115+
usleep(self::DEFAULT_SLEEP_TIMEOUT);
116+
} while ($iterator > 0);
117+
118+
return $evictedKeys;
119+
}
120+
}

‎composer.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
"require": {
77
"php": "^7.0",
88
"ext-json": "*",
9+
"colinmollenhour/cache-backend-redis": "^1.9",
10+
"colinmollenhour/credis": "^1.6",
911
"newrelic/monolog-enricher": "^1.0"
1012
},
1113
"suggest": {
@@ -21,6 +23,9 @@
2123
"phpunit/phpunit": "^6.2",
2224
"squizlabs/php_codesniffer": "^3.0"
2325
},
26+
"config": {
27+
"sort-packages": true
28+
},
2429
"scripts": {
2530
"test": [
2631
"@phpstan",

‎etc/crontab.xml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
/**
4+
* Copyright © Magento, Inc. All rights reserved.
5+
* See COPYING.txt for license details.
6+
*/
7+
-->
8+
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:module:Magento_Cron:etc/crontab.xsd">
9+
<group id="cache_evict">
10+
<job name="cache_evict_keys" instance="Magento\CloudComponents\Cron\Evict" method="execute">
11+
<schedule>0 */12 * * *</schedule>
12+
</job>
13+
</group>
14+
</config>

‎etc/di.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<item name="configShowStoreUrlCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowStoreUrlCommand</item>
1313
<item name="configShowEntityUrlsCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowEntityUrlsCommand</item>
1414
<item name="ConfigShowDefaultUrlCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowDefaultUrlCommand</item>
15+
<item name="CacheEvict" xsi:type="object">Magento\CloudComponents\Console\Command\CacheEvict</item>
1516
</argument>
1617
</arguments>
1718
</type>

0 commit comments

Comments
 (0)