Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/Asset/AssetIndexer.php
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,8 @@ public function processIndexSession(AssetIndexingSession $indexingSession): Asse
return;
}

if ($indexEntry) {
$indexEntry->transitionTo(AssetIndexStatus::Processing);
if ($indexEntry && ! $indexEntry->claim()) {
$indexEntry = null;
}
});

Expand Down Expand Up @@ -462,9 +462,8 @@ private function folderAssetCountQuery(AssetIndexingSession $session, bool $miss

public function getNextIndexEntry(AssetIndexingSession $session): ?AssetIndexData
{
return AssetIndexData::query()
return AssetIndexData::processingClaim()->whereClaimable(AssetIndexData::query())
->where('sessionId', $session->id)
->where('status', AssetIndexStatus::Pending)
->orderBy('id')
->first();
}
Expand Down
25 changes: 25 additions & 0 deletions src/Asset/Models/AssetIndexData.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace CraftCms\Cms\Asset\Models;

use CraftCms\Cms\Asset\Enums\AssetIndexStatus;
use CraftCms\Cms\Database\Claim;
use CraftCms\Cms\Database\Table;
use CraftCms\Cms\Shared\BaseModel;
use CraftCms\Cms\Shared\Concerns\HasUid;
Expand Down Expand Up @@ -51,6 +52,30 @@ public function __toString(): string
return (string) $this->uri;
}

/**
* Entries that have been processing for more than 5 minutes count as abandoned, so another worker can claim them.
*/
public static function processingClaim(): Claim
{
return new Claim('status', AssetIndexStatus::Processing, AssetIndexStatus::Pending, ttl: 300);
}

/**
* Moves a pending entry, or one abandoned while processing, to processing.
*
* @return bool Whether this call claimed the entry. `false` means another worker is processing it, or it has finished.
*/
public function claim(): bool
{
if (! self::processingClaim()->claim(static::query()->whereKey($this->getKey()))) {
return false;
}

$this->refresh();

return true;
}

public function transitionTo(AssetIndexStatus $status, ?int $recordId = null): void
{
$currentStatus = $this->status;
Expand Down
2 changes: 1 addition & 1 deletion src/Cms.php
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

public const string VERSION = '6.0.0-beta.1';

public const string SCHEMA_VERSION = '6.0.0.40';
public const string SCHEMA_VERSION = '6.0.0.41';

public const string MIN_VERSION_REQUIRED = '5.9.0';

Expand Down
98 changes: 98 additions & 0 deletions src/Database/Claim.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
<?php

declare(strict_types=1);

namespace CraftCms\Cms\Database;

use CraftCms\Cms\Support\Query;
use DateTimeInterface;
use Illuminate\Database\Eloquent\Builder as EloquentBuilder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Query\Builder;

/**
* A row that one worker claims at a time. A claim older than its TTL counts as abandoned.
*
* A worker that is killed or hits a fatal error never releases its claim, so the row stays
* claimed. Once the claim is older than the TTL, another worker can claim the row.
*
* @since 6.0.0
*/
readonly class Claim
{
/**
* @param string $column The column holding the claim state
* @param mixed $claimed The column value of a claimed row
* @param mixed $unclaimed The column value of a row that no worker has claimed
* @param int $ttl Seconds after which a claim counts as abandoned
* @param string $timestampColumn The column holding the time the row was claimed
*/
public function __construct(
public string $column,
public mixed $claimed,
public mixed $unclaimed,
public int $ttl,
public string $timestampColumn = 'dateUpdated',
) {}

/**
* Limits the query to unclaimed rows and rows with abandoned claims.
*
* @template TQuery of Builder|EloquentBuilder
*
* @param TQuery $query
* @return TQuery
*/
public function whereClaimable(Builder|EloquentBuilder $query): Builder|EloquentBuilder
{
return $query->where(fn (\Illuminate\Contracts\Database\Query\Builder $query) => $query
->where($this->column, $this->unclaimed)
->orWhere(fn (\Illuminate\Contracts\Database\Query\Builder $query) => $query
->where($this->column, $this->claimed)
->where(fn (\Illuminate\Contracts\Database\Query\Builder $query) => $query
->whereNull($this->timestampColumn)
->orWhere($this->timestampColumn, '<', $this->cutoff()))));
}

/**
* Limits the query to rows with a live claim.
*
* @template TQuery of Builder|EloquentBuilder
*
* @param TQuery $query
* @return TQuery
*/
public function whereClaimed(Builder|EloquentBuilder $query): Builder|EloquentBuilder
{
return $query
->where($this->column, $this->claimed)
->where($this->timestampColumn, '>=', $this->cutoff());
}

/**
* Claims the unclaimed and abandoned rows the query matches, with one conditional update.
*
* @param Builder|EloquentBuilder<covariant Model> $query
* @return bool Whether this call claimed any rows. `false` means another worker holds a live claim, or the rows no longer exist.
*/
public function claim(Builder|EloquentBuilder $query): bool
{
return $this->whereClaimable($query)->update([
$this->column => $this->claimed,
$this->timestampColumn => Query::prepareDateForDb(now()),
]) > 0;
}

/**
* Returns whether a claim made at the given time has been abandoned.
*/
public function isAbandoned(?DateTimeInterface $claimedAt): bool
{
return $claimedAt === null || now()->diffInSeconds($claimedAt, true) > $this->ttl;
}

private function cutoff(): string
{
return Query::prepareDateForDb(now()->subSeconds($this->ttl));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<?php

declare(strict_types=1);

use CraftCms\Cms\Database\Table;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;

return new class extends Migration
{
public function up(): void
{
if (Schema::hasColumn(Table::SEARCHINDEXQUEUE, 'dateReserved')) {
return;
}

// Reserved rows without a date count as abandoned, so the next index job picks them up again.
Schema::table(Table::SEARCHINDEXQUEUE, function (Blueprint $table): void {
$table->dateTime('dateReserved')->nullable()->after('reserved');
});
}

public function down(): void
{
Schema::table(Table::SEARCHINDEXQUEUE, function (Blueprint $table): void {
$table->dropColumn('dateReserved');
});
}
};
1 change: 1 addition & 0 deletions src/Database/Migrations/Install.php
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,7 @@ public function createTables(?Logger $logger = null): void
$table->integer('elementId');
$table->integer('siteId');
$table->boolean('reserved')->default(false);
$table->dateTime('dateReserved')->nullable();
});

$logger?->subLabel('searchindexqueue_fields');
Expand Down
2 changes: 1 addition & 1 deletion src/Http/Middleware/RunQueue.php
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ private function shouldInjectQueueRunner(Request $request, Response $response):
return false;
}

if ($this->progressService->getActive()->isNotEmpty()) {
if ($this->progressService->hasReservedJobs()) {
return false;
}

Expand Down
35 changes: 33 additions & 2 deletions src/Image/Data/ImageTransformIndex.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace CraftCms\Cms\Image\Data;

use CraftCms\Cms\Component\Component;
use CraftCms\Cms\Image\ImageTransformer;
use CraftCms\Cms\Image\ImageTransformHelper;
use DateTimeInterface;

Expand Down Expand Up @@ -50,16 +51,46 @@ public function __construct(array|object $config = [])
{
parent::__construct($config);

// Reset inProgress if stale (30+ seconds)
// Reset inProgress if the request generating the transform abandoned it
if (
$this->inProgress
&& $this->dateUpdated
&& now()->diffInSeconds($this->dateUpdated, true) > 30
&& ImageTransformer::inProgressClaim()->isAbandoned($this->dateUpdated)
) {
$this->inProgress = false;
}
}

/**
* Marks the transform as being generated. A failure from an earlier attempt stays recorded
* until this attempt succeeds or fails.
*/
public function markGenerating(): void
{
$this->inProgress = true;
$this->fileExists = false;
}

/**
* Marks the transform as generated.
*/
public function markGenerated(): void
{
$this->inProgress = false;
$this->fileExists = true;
$this->error = false;
}

/**
* Marks the transform as failed to generate.
*/
public function markFailed(): void
{
$this->inProgress = false;
$this->fileExists = false;
$this->error = true;
}

public function getTransform(): ImageTransform
{
if ($this->_transform === null) {
Expand Down
70 changes: 42 additions & 28 deletions src/Image/ImageTransformer.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
use CraftCms\Cms\Asset\Elements\Asset;
use CraftCms\Cms\Asset\Exceptions\ImageTransformException;
use CraftCms\Cms\Cms;
use CraftCms\Cms\Database\Claim;
use CraftCms\Cms\Database\Table;
use CraftCms\Cms\Filesystem\Exceptions\FilesystemException;
use CraftCms\Cms\Image\Data\ImageTransform;
Expand Down Expand Up @@ -132,47 +133,35 @@ public function getTransformUrl(
: $this->privateTransformUrl($index);
}

// Is the transform being generated by another request?
if ($index->inProgress) {
for ($try = 1; $try <= 30; $try++) {
if ($index->error) {
throw new ImageTransformException(t('Failed to generate transform with id of {id}.', [
'id' => $index->id,
]));
}

// Wait a second and check again
maxPowerCaptain();
Sleep::sleep(1);
$index = $this->getTransformIndexModelById($index->id);
if (! $index->inProgress) {
break;
}
// Claim the transform, or wait while another request generates it.
// If that request fails or dies, its claim ends and this request generates the transform.
for ($try = 1; ! $index->fileExists && ! $this->claimTransformIndex($index); $try++) {
if ($try > 60) {
throw new ImageTransformException(t('Failed to generate transform with id of {id}.', [
'id' => $index->id,
]));
}

// Wait a second and check again
maxPowerCaptain();
Sleep::sleep(1);
$index = $this->getTransformIndexModelById($index->id);
}

// No file, then
if (! $index->fileExists) {
// Mark the transform as in progress
$index->inProgress = true;
$this->storeTransformIndexData($index);

// Generate the transform
try {
$this->generateTransform($index, $asset, $transformer);
} catch (Exception $e) {
$index->inProgress = false;
$index->fileExists = false;
$index->error = true;
$index->markFailed();
$this->storeTransformIndexData($index);

throw new ImageTransformException(t('Failed to generate transform with id of {id}.', [
'id' => $index->id,
]), previous: $e);
}

$index->inProgress = false;
$index->fileExists = true;
$index->markGenerated();
$this->storeTransformIndexData($index);
}
}
Expand Down Expand Up @@ -636,16 +625,41 @@ public function storeTransformIndexData(ImageTransformIndex $index): void
*/
public function getPendingTransformIndexIds(): array
{
return $this->createTransformIndexQuery()
return self::inProgressClaim()->whereClaimable($this->createTransformIndexQuery())
->where([
'fileExists' => false,
'inProgress' => false,
'error' => false,
])
->pluck('id')
->all();
}

/**
* Transforms that have been in progress for more than 30 seconds count as abandoned.
*/
public static function inProgressClaim(): Claim
{
return new Claim('inProgress', true, false, ttl: 30);
}

/**
* Marks a transform as in progress, unless another request is already generating it.
*/
private function claimTransformIndex(ImageTransformIndex $index): bool
{
$query = $this->createTransformIndexQuery()
->where('id', $index->id)
->where('fileExists', false);

if (! self::inProgressClaim()->claim($query)) {
return false;
}

$index->markGenerating();

return true;
}

public function getTransformIndexModelById(int $transformId): ?ImageTransformIndex
{
$result = $this->createTransformIndexQuery()
Expand Down
Loading
Loading