diff --git a/UPGRADE-6.5.md b/UPGRADE-6.5.md index 48fff269579..c45677be765 100644 --- a/UPGRADE-6.5.md +++ b/UPGRADE-6.5.md @@ -497,12 +497,10 @@ shopware: cache: invalidation: delay_options: - storage: redis + storage: cache dsn: 'redis://localhost' ``` -Since 6.6.10.0 we also have a MySQL implementation available: `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage`. Use it via `mysql` - # 6.5.5.0 Shopware 6.5 introduces a new more flexible stock management system. Please see the [ADR](adr/2023-05-15-stock-api.md) for a more detailed description of the why & how. diff --git a/UPGRADE-6.6.md b/UPGRADE-6.6.md index 902c679c5ab..56466da8743 100644 --- a/UPGRADE-6.6.md +++ b/UPGRADE-6.6.md @@ -854,12 +854,10 @@ shopware: cache: invalidation: delay_options: - storage: redis + storage: cache dsn: 'redis://localhost' ``` -Since 6.6.10.0 we also have a MySQL implementation available: `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage`. Use it via `mysql` - # General Core Breaking Changes ## Symfony 7 upgrade @@ -2070,12 +2068,10 @@ shopware: cache: invalidation: delay_options: - storage: redis + storage: cache dsn: 'redis://localhost' ``` -Since 6.6.10.0 we also have a MySQL implementation available: `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage`. Use it via `mysql` - ## Introduced in 6.5.5.0 ## New stock handling implementation is now the default diff --git a/changelog/_unreleased/2024-12-09-add-mysql-cache-invalidator-storage.md b/changelog/_unreleased/2024-12-09-add-mysql-cache-invalidator-storage.md deleted file mode 100644 index 541682bc7e2..00000000000 --- a/changelog/_unreleased/2024-12-09-add-mysql-cache-invalidator-storage.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Add mysql cache invalidator storage -issue: NEXT-39316 ---- -# Core -* Added `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage` to collect invalidations in MySQL in an atomic operation. -___ -# Upgrade Information - -## Addition of MySQLInvalidatorStorage - -We added a new MySQL cache invalidator storage so you can take advantage of delayed cache invalidation without needing Redis (Redis is still preferred). - -```yaml -shopware: - cache: - invalidation: - delay: 1 - delay_options: - storage: mysql -``` diff --git a/changelog/release-6-5-6-0/2023-09-14-collect-cache-invalidations-in-redis.md b/changelog/release-6-5-6-0/2023-09-14-collect-cache-invalidations-in-redis.md index 6cbffaefdaa..c3d54417b09 100644 --- a/changelog/release-6-5-6-0/2023-09-14-collect-cache-invalidations-in-redis.md +++ b/changelog/release-6-5-6-0/2023-09-14-collect-cache-invalidations-in-redis.md @@ -21,12 +21,10 @@ shopware: cache: invalidation: delay_options: - storage: redis + storage: cache dsn: 'redis://localhost' ``` -Since 6.6.10.0 we also have a MySQL implementation available: `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage`. Use it via `mysql` - ___ # Next Major Version Changes @@ -42,8 +40,6 @@ shopware: cache: invalidation: delay_options: - storage: redis + storage: cache dsn: 'redis://localhost' ``` - -Since 6.6.10.0 we also have a MySQL implementation available: `\Shopware\Core\Framework\Adapter\Cache\InvalidatorStorage\MySQLInvalidatorStorage`. Use it via `mysql` diff --git a/config-schema.json b/config-schema.json index f3be682f48a..7d1be0e0cfc 100644 --- a/config-schema.json +++ b/config-schema.json @@ -584,11 +584,10 @@ "properties": { "storage": { "type": "string", - "default": "mysql", + "default": "cache", "enum": [ "cache", - "redis", - "mysql" + "redis" ] }, "dsn": { diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index aa0968f5fda..ea10f7ff7d9 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -2245,6 +2245,11 @@ parameters: count: 1 path: src/Core/Framework/DataAbstractionLayer/Dbal/QueryBuilder.php + - + message: "#^Throwing new exceptions within classes are not allowed\\. Please use domain exception pattern\\. See https\\://github\\.com/shopware/platform/blob/v6\\.4\\.20\\.0/adr/2022\\-02\\-24\\-domain\\-exceptions\\.md$#" + count: 1 + path: src/Core/Framework/DataAbstractionLayer/Doctrine/MultiInsertQueryQueue.php + - message: "#^Throwing new exceptions within classes are not allowed\\. Please use domain exception pattern\\. See https\\://github\\.com/shopware/platform/blob/v6\\.4\\.20\\.0/adr/2022\\-02\\-24\\-domain\\-exceptions\\.md$#" count: 3 diff --git a/src/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorage.php b/src/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorage.php deleted file mode 100644 index 9c8aacfa69f..00000000000 --- a/src/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorage.php +++ /dev/null @@ -1,115 +0,0 @@ -debug = $debug ?? (fn () => null)(...); - } - - public function store(array $tags): void - { - if (empty($tags)) { - return; - } - - $insertQueue = new MultiInsertQueryQueue($this->connection, chunkSize: 1000, ignoreErrors: true); - $insertQueue->addInserts( - self::TABLE_NAME, - array_map( - fn (string $tag) => ['id' => Uuid::randomBytes(), 'tag' => $tag], - array_values($tags) - ) - ); - - // we execute in read committed isolation so that row gap locks are not applied when inserting - // this helps us prevent locks when trying to insert duplicate tags. - $this->readCommittedIsolation(fn () => $insertQueue->execute()); - } - - /** - * We attempt to read and lock all rows in the table. This works fine for subsequent executions, e.g. calling this method multiple times synchronously - * however, if running in parallel, it can still be that the database returns similar result sets when it did not get a chance to lock the rows. Then, when trying to - * delete, locks can be encountered if two processes attempt to delete the same row. - */ - public function loadAndDelete(): array - { - try { - return $this->readCommittedIsolation($this->executeLoadAndDelete(...)); - } catch (\Throwable $e) { - $this->logger->warning('Cache tags could not be fetched or removed from storage. Possible deadlock encountered. If the error persists, try the redis adapter. Error: ' . $e->getMessage()); - - return []; - } - } - - /** - * @return list - */ - private function executeLoadAndDelete(): array - { - // fetch and lock records, ignoring locked records, se we don't handle tags - // being processed by parallel worker - $rows = $this->connection->fetchAllAssociative( - \sprintf('SELECT id, tag FROM %s ORDER BY id FOR UPDATE SKIP LOCKED', self::TABLE_NAME) - ); - - ($this->debug)($this, $rows); - - if (empty($rows)) { - return []; - } - - $firstTagId = $rows[0]['id']; - $lastTagId = $rows[array_key_last($rows)]['id']; - - $query = new RetryableQuery( - $this->connection, - $this->connection->prepare(\sprintf('DELETE FROM %s WHERE id BETWEEN ? AND ?', self::TABLE_NAME)) - ); - $query->execute([$firstTagId, $lastTagId]); - - return array_column($rows, 'tag'); - } - - /** - * @param \Closure(Connection):T $callback - * - * @return T - * - * @template T - */ - private function readCommittedIsolation(\Closure $callback): mixed - { - // used so that we don't lock the table for inserts - $transactionIsolation = $this->connection->getTransactionIsolation(); - $this->connection->setTransactionIsolation(TransactionIsolationLevel::READ_COMMITTED); - - try { - return $this->connection->transactional($callback); - } finally { - // restore original isolation mode - $this->connection->setTransactionIsolation($transactionIsolation); - } - } -} diff --git a/src/Core/Framework/DataAbstractionLayer/DataAbstractionLayerException.php b/src/Core/Framework/DataAbstractionLayer/DataAbstractionLayerException.php index fa1bcaf2a5a..710d438d240 100644 --- a/src/Core/Framework/DataAbstractionLayer/DataAbstractionLayerException.php +++ b/src/Core/Framework/DataAbstractionLayer/DataAbstractionLayerException.php @@ -82,7 +82,6 @@ class DataAbstractionLayerException extends HttpException public const PRIMARY_KEY_NOT_PROVIDED = 'FRAMEWORK__PRIMARY_KEY_NOT_PROVIDED'; public const NO_GENERATOR_FOR_FIELD_TYPE = 'FRAMEWORK__NO_GENERATOR_FOR_FIELD_TYPE'; public const FOREIGN_KEY_NOT_FOUND_IN_DEFINITION = 'FRAMEWORK__FOREIGN_KEY_NOT_FOUND_IN_DEFINITION'; - public const INVALID_CHUNK_SIZE = 'FRAMEWORK__INVALID_CHUNK_SIZE'; public static function invalidSerializerField(string $expectedClass, Field $field): self { @@ -773,14 +772,4 @@ public static function foreignKeyNotFoundInDefinition(string $association, strin ['association' => $association, 'entityDefinition' => $entityDefinition] ); } - - public static function invalidChunkSize(int $size): self - { - return new self( - Response::HTTP_INTERNAL_SERVER_ERROR, - self::INVALID_CHUNK_SIZE, - 'Parameter $chunkSize needs to be a positive integer starting with 1, "{{ size }}" given', - ['size' => $size] - ); - } } diff --git a/src/Core/Framework/DataAbstractionLayer/DefinitionValidator.php b/src/Core/Framework/DataAbstractionLayer/DefinitionValidator.php index 4f275a64fb2..3c414a909a4 100644 --- a/src/Core/Framework/DataAbstractionLayer/DefinitionValidator.php +++ b/src/Core/Framework/DataAbstractionLayer/DefinitionValidator.php @@ -100,7 +100,6 @@ class DefinitionValidator 'refresh_token', 'usage_data_entity_deletion', 'one_time_tasks', - 'invalidation_tags', ]; private const IGNORED_ENTITY_PROPERTIES = [ diff --git a/src/Core/Framework/DataAbstractionLayer/Doctrine/MultiInsertQueryQueue.php b/src/Core/Framework/DataAbstractionLayer/Doctrine/MultiInsertQueryQueue.php index e7571148893..264f7b4cdc3 100644 --- a/src/Core/Framework/DataAbstractionLayer/Doctrine/MultiInsertQueryQueue.php +++ b/src/Core/Framework/DataAbstractionLayer/Doctrine/MultiInsertQueryQueue.php @@ -4,7 +4,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\ParameterType; -use Shopware\Core\Framework\DataAbstractionLayer\DataAbstractionLayerException; use Shopware\Core\Framework\DataAbstractionLayer\Dbal\EntityDefinitionQueryHelper; use Shopware\Core\Framework\Log\Package; @@ -33,7 +32,9 @@ public function __construct( private readonly bool $useReplace = false ) { if ($chunkSize < 1) { - throw DataAbstractionLayerException::invalidChunkSize($chunkSize); + throw new \InvalidArgumentException( + \sprintf('Parameter $chunkSize needs to be a positive integer starting with 1, "%d" given', $chunkSize) + ); } $this->chunkSize = $chunkSize; } @@ -69,17 +70,6 @@ public function addInsert(string $table, array $data, ?array $types = null): voi ]; } - /** - * @param list> $rows - * @param array|null $types - */ - public function addInserts(string $table, array $rows, ?array $types = null): void - { - foreach ($rows as $row) { - $this->addInsert($table, $row, $types); - } - } - public function execute(): void { if (empty($this->inserts)) { diff --git a/src/Core/Framework/DataAbstractionLayer/Doctrine/RetryableQuery.php b/src/Core/Framework/DataAbstractionLayer/Doctrine/RetryableQuery.php index 41ca2fa2f1b..f904f2e3360 100644 --- a/src/Core/Framework/DataAbstractionLayer/Doctrine/RetryableQuery.php +++ b/src/Core/Framework/DataAbstractionLayer/Doctrine/RetryableQuery.php @@ -19,7 +19,7 @@ public function __construct( } /** - * @param array $params + * @param array $params */ public function execute(array $params = []): int { diff --git a/src/Core/Framework/DependencyInjection/cache.xml b/src/Core/Framework/DependencyInjection/cache.xml index f1003785a45..1e32b688c74 100644 --- a/src/Core/Framework/DependencyInjection/cache.xml +++ b/src/Core/Framework/DependencyInjection/cache.xml @@ -35,13 +35,6 @@ - - - - - - - diff --git a/src/Core/Framework/Resources/config/packages/shopware.yaml b/src/Core/Framework/Resources/config/packages/shopware.yaml index 31fec5b36ba..5ad6a7c53fb 100644 --- a/src/Core/Framework/Resources/config/packages/shopware.yaml +++ b/src/Core/Framework/Resources/config/packages/shopware.yaml @@ -511,7 +511,7 @@ shopware: invalidation: delay: 0 delay_options: - storage: mysql + storage: redis http_cache: ['logged-in', 'cart-filled'] product_listing_route: [] product_detail_route: [] diff --git a/src/Core/Migration/V6_6/Migration1733745893createTagStorageTable.php b/src/Core/Migration/V6_6/Migration1733745893createTagStorageTable.php deleted file mode 100644 index 801f08ca297..00000000000 --- a/src/Core/Migration/V6_6/Migration1733745893createTagStorageTable.php +++ /dev/null @@ -1,29 +0,0 @@ -executeStatement(' - CREATE TABLE IF NOT EXISTS invalidation_tags ( - id BINARY(16) NOT NULL PRIMARY KEY, - tag VARCHAR(255) NOT NULL UNIQUE - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - '); - } -} diff --git a/tests/integration/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorageTest.php b/tests/integration/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorageTest.php deleted file mode 100644 index d94e2f0a934..00000000000 --- a/tests/integration/Core/Framework/Adapter/Cache/InvalidatorStorage/MySQLInvalidatorStorageTest.php +++ /dev/null @@ -1,194 +0,0 @@ -connection = $this->getContainer()->get(Connection::class); - $this->logger = $this->createMock(LoggerInterface::class); - - $this->storage = new MySQLInvalidatorStorage($this->connection, $this->logger); - } - - protected function tearDown(): void - { - parent::setUp(); - - $this->connection->executeStatement('DELETE FROM invalidation_tags'); - } - - public function testStoreSingleTag(): void - { - $this->storage->store(['tag1']); - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertSame(['tag1'], $result); - } - - public function testStoreMultipleTags(): void - { - $this->storage->store(['tag1', 'tag2', 'tag3']); - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertSame(['tag1', 'tag2', 'tag3'], $result); - } - - public function testStoreNoTags(): void - { - $this->storage->store([]); - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertEmpty($result); - } - - public function testLoadAndDeleteSingleTag(): void - { - $this->storage->store(['tag1']); - $result = $this->storage->loadAndDelete(); - - static::assertSame(['tag1'], $result); - $remaining = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - static::assertEmpty($remaining); - } - - public function testLoadAndDeleteMultipleTags(): void - { - $this->storage->store(['tag1', 'tag2']); - $result = $this->storage->loadAndDelete(); - - static::assertSame(['tag1', 'tag2'], $result); - $remaining = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - static::assertEmpty($remaining); - } - - public function testLoadAndDeleteWhenEmpty(): void - { - $result = $this->storage->loadAndDelete(); - static::assertEmpty($result); - } - - public function testStoreDuplicateTags(): void - { - $this->storage->store(['tag1', 'tag1', 'tag2']); - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertSame(['tag1', 'tag2'], $result); - } - - public function testLoadAndDeleteOnlyDeletesSelectedItems(): void - { - $storage = new MySQLInvalidatorStorage( - $this->connection, - $this->logger, - fn (MySQLInvalidatorStorage $storage, array $tags) => $storage->store(['tag4', 'tag5', 'tag6']), - ); - - // store these first - $this->storage->store(['tag1', 'tag2', 'tag3']); - - $tags = $storage->loadAndDelete(); - - static::assertEquals(['tag1', 'tag2', 'tag3'], $tags); - - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertSame(['tag4', 'tag5', 'tag6'], $result); - } - - public function testLoadAndDeleteWithParallelDelete(): void - { - // create a separate connection to simulate parallel request - $connection2 = MySQLFactory::create(); - $storage2 = new MySQLInvalidatorStorage($connection2, $this->logger); - - $storage1 = new MySQLInvalidatorStorage( - $this->connection, - $this->logger, - function () use ($storage2): void { - // in the middle of the original request running `loadAndDelete` - // 3. insert some more tags (in parallel worker) - $storage2->store(['tag4', 'tag5', 'tag6']); - $storage2->store(['tag7', 'tag8', 'tag9']); - - // 4. now load and delete (in parallel worker) - // should only load and delete our tags inserted in this process - // as other tags will be locked by parallel worker (first worker, step 2) - $delete = $storage2->loadAndDelete(); - - static::assertSame(['tag4', 'tag5', 'tag6', 'tag7', 'tag8', 'tag9'], $delete); - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - static::assertSame(['tag1', 'tag2', 'tag3'], $result); - }, - ); - - // 1. store these first - $storage1->store(['tag1', 'tag2', 'tag3']); - - // 2. load tags on original connection (which will trigger callable from above to simulate parallel request loading tags) - $tags = $storage1->loadAndDelete(); - - static::assertEquals(['tag1', 'tag2', 'tag3'], $tags); - - $result = $this->connection->fetchFirstColumn('SELECT tag FROM invalidation_tags'); - - static::assertSame([], $result); - } - - public function testLoadAndDeleteExceptionIsCaughtAndLogged(): void - { - $this->logger->expects(static::once())->method('warning') - ->with('Cache tags could not be fetched or removed from storage. Possible deadlock encountered. If the error persists, try the redis adapter. Error: Deadlock'); - - $connection = $this->createMock(Connection::class); - - $connection->expects(static::once()) - ->method('fetchAllAssociative') - ->willReturn([['id' => 'id1', 'tag1'], ['id' => 'id2', 'tag2']]); - - $statement = $this->createMock(Statement::class); - - $e = new class('Deadlock') extends \Exception implements RetryableException {}; - - $statement - ->method('executeStatement') - ->with(['id1', 'id2']) - ->willThrowException($e); - - $connection->expects(static::once()) - ->method('prepare') - ->with('DELETE FROM invalidation_tags WHERE id BETWEEN ? AND ?') - ->willReturn($statement); - - $connection->expects(static::once()) - ->method('transactional') - ->willReturnCallback(fn (callable $cb) => $cb()); - - $storage = new MySQLInvalidatorStorage($connection, $this->logger); - $storage->loadAndDelete(); - } -} diff --git a/tests/integration/Core/Framework/DataAbstractionLayer/Doctrine/Doctrine/MultiInsertQueryQueueTest.php b/tests/integration/Core/Framework/DataAbstractionLayer/Doctrine/Doctrine/MultiInsertQueryQueueTest.php index 63cb299c2e3..bbd133fe078 100644 --- a/tests/integration/Core/Framework/DataAbstractionLayer/Doctrine/Doctrine/MultiInsertQueryQueueTest.php +++ b/tests/integration/Core/Framework/DataAbstractionLayer/Doctrine/Doctrine/MultiInsertQueryQueueTest.php @@ -2,7 +2,6 @@ namespace Shopware\Tests\Integration\Core\Framework\DataAbstractionLayer\Doctrine\Doctrine; -use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; use PHPUnit\Framework\TestCase; use Shopware\Core\Content\Category\CategoryDefinition; @@ -98,42 +97,4 @@ public function testAddUpdateFieldOnDuplicateKey(): void static::assertNotFalse($type); static::assertSame(CategoryDefinition::TYPE_FOLDER, $type); } - - public function testAddInserts(): void - { - $catA = Uuid::randomBytes(); - $catB = Uuid::randomBytes(); - - $date = (new \DateTime())->format(Defaults::STORAGE_DATE_TIME_FORMAT); - - $inserts = [ - [ - 'id' => $catA, - 'version_id' => Uuid::fromHexToBytes(Defaults::LIVE_VERSION), - 'type' => CategoryDefinition::TYPE_LINK, - 'created_at' => $date, - 'updated_at' => null, - ], - [ - 'id' => $catB, - 'version_id' => Uuid::fromHexToBytes(Defaults::LIVE_VERSION), - 'type' => CategoryDefinition::TYPE_LINK, - 'created_at' => $date, - 'updated_at' => null, - ], - ]; - - $connection = static::getContainer()->get(Connection::class); - $query = new MultiInsertQueryQueue($connection); - $query->addInserts('category', $inserts); - $query->execute(); - - $rows = $connection->fetchFirstColumn( - 'SELECT id FROM `category` WHERE id IN(:ids)', - ['ids' => [$catA, $catB]], - ['ids' => ArrayParameterType::BINARY] - ); - - static::assertSame([$catA, $catB], $rows); - } } diff --git a/tests/migration/Core/V6_6/Migration1733745893createTagStorageTableTest.php b/tests/migration/Core/V6_6/Migration1733745893createTagStorageTableTest.php deleted file mode 100644 index 0ea3c6f835d..00000000000 --- a/tests/migration/Core/V6_6/Migration1733745893createTagStorageTableTest.php +++ /dev/null @@ -1,52 +0,0 @@ -connection = KernelLifecycleManager::getConnection(); - - $this->connection->executeStatement('DROP TABLE IF EXISTS `invalidation_tags`;'); - } - - public function testGetCreationTimestamp(): void - { - $migration = new Migration1733745893createTagStorageTable(); - static::assertSame(1733745893, $migration->getCreationTimestamp()); - } - - public function testTableIsCreated(): void - { - $sm = $this->connection->createSchemaManager(); - - static::assertFalse($sm->tablesExist('invalidation_tags')); - - $migration = new Migration1733745893createTagStorageTable(); - - $migration->update($this->connection); - $migration->update($this->connection); - - static::assertTrue($sm->tablesExist('invalidation_tags')); - - $cols = $sm->listTableColumns('invalidation_tags'); - static::assertCount(2, $cols); - static::assertSame('tag', $cols['tag']->getName()); - static::assertSame('id', $cols['id']->getName()); - } -}