PHPORM-28 Add Scout engine to index into MongoDB Search (#3205)

Author: GromNaN

.github/workflows/build-ci.yml

@@ -65,6 +65,9 @@ jobs:
                     until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.runCommand({ ping: 1 })"; do
                       sleep 1
                     done
+                    until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.createCollection('connection_test') && db.getCollection('connection_test').createSearchIndex({mappings:{dynamic: true}})"; do
+                      sleep 1
+                    done
 
             -   name: "Show MongoDB server status"
                 run: |

composer.json

@@ -35,6 +35,7 @@
     },
     "require-dev": {
         "mongodb/builder": "^0.2",
+        "laravel/scout": "^11",
         "league/flysystem-gridfs": "^3.28",
         "league/flysystem-read-only": "^3.0",
         "phpunit/phpunit": "^10.3",

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
     app:
         tty: true
@@ -16,11 +14,11 @@ services:
 
     mongodb:
         container_name: mongodb
-        image: mongo:latest
+        image: mongodb/mongodb-atlas-local:latest
         ports:
             - "27017:27017"
         healthcheck:
-            test: echo 'db.runCommand("ping").ok' | mongosh mongodb:27017 --quiet
+            test: mongosh --quiet --eval 'db.runCommand("ping").ok'
             interval: 10s
             timeout: 10s
             retries: 5

phpstan-baseline.neon

@@ -24,3 +24,8 @@ parameters:
 			message: "#^Method Illuminate\\\\Database\\\\Schema\\\\Blueprint\\:\\:create\\(\\) invoked with 1 parameter, 0 required\\.$#"
 			count: 1
 			path: src/Schema/Builder.php
+
+		-
+			message: "#^Call to an undefined method Illuminate\\\\Support\\\\HigherOrderCollectionProxy\\<\\(int\\|string\\), Illuminate\\\\Database\\\\Eloquent\\\\Model\\>\\:\\:pushSoftDeleteMetadata\\(\\)\\.$#"
+			count: 1
+			path: src/Scout/ScoutEngine.php

phpunit.xml.dist

@@ -17,7 +17,7 @@
         </testsuite>
     </testsuites>
     <php>
-        <env name="MONGODB_URI" value="mongodb://mongodb/"/>
+        <env name="MONGODB_URI" value="mongodb://mongodb/?directConnection=true"/>
         <env name="MONGODB_DATABASE" value="unittest"/>
         <env name="SQLITE_DATABASE" value=":memory:"/>
         <env name="QUEUE_CONNECTION" value="database"/>

src/MongoDBServiceProvider.php

@@ -7,19 +7,22 @@
 use Closure;
 use Illuminate\Cache\CacheManager;
 use Illuminate\Cache\Repository;
+use Illuminate\Container\Container;
 use Illuminate\Filesystem\FilesystemAdapter;
 use Illuminate\Filesystem\FilesystemManager;
 use Illuminate\Foundation\Application;
 use Illuminate\Session\SessionManager;
 use Illuminate\Support\ServiceProvider;
 use InvalidArgumentException;
+use Laravel\Scout\EngineManager;
 use League\Flysystem\Filesystem;
 use League\Flysystem\GridFS\GridFSAdapter;
 use League\Flysystem\ReadOnly\ReadOnlyFilesystemAdapter;
 use MongoDB\GridFS\Bucket;
 use MongoDB\Laravel\Cache\MongoStore;
 use MongoDB\Laravel\Eloquent\Model;
 use MongoDB\Laravel\Queue\MongoConnector;
+use MongoDB\Laravel\Scout\ScoutEngine;
 use RuntimeException;
 use Symfony\Component\HttpFoundation\Session\Storage\Handler\MongoDbSessionHandler;
 
@@ -102,6 +105,7 @@ public function register()
         });
 
         $this->registerFlysystemAdapter();
+        $this->registerScoutEngine();
     }
 
     private function registerFlysystemAdapter(): void
@@ -155,4 +159,21 @@ private function registerFlysystemAdapter(): void
             });
         });
     }
+
+    private function registerScoutEngine(): void
+    {
+        $this->app->resolving(EngineManager::class, function (EngineManager $engineManager) {
+            $engineManager->extend('mongodb', function (Container $app) {
+                $connectionName = $app->get('config')->get('scout.mongodb.connection', 'mongodb');
+                $connection = $app->get('db')->connection($connectionName);
+                $softDelete = (bool) $app->get('config')->get('scout.soft_delete', false);
+
+                assert($connection instanceof Connection, new InvalidArgumentException(sprintf('The connection "%s" is not a MongoDB connection.', $connectionName)));
+
+                return new ScoutEngine($connection->getMongoDB(), $softDelete);
+            });
+
+            return $engineManager;
+        });
+    }
 }

src/Scout/ScoutEngine.php

@@ -406,8 +406,9 @@ public function testSoftDelete(): void
         $this->assertEquals(2, Soft::count());
     }
 
+    /** @param class-string<Model> $model */
     #[DataProvider('provideId')]
-    public function testPrimaryKey(string $model, $id, $expected, bool $expectedFound): void
+    public function testPrimaryKey(string $model, mixed $id, mixed $expected, bool $expectedFound): void
     {
         $model::truncate();
         $expectedType = get_debug_type($expected);

tests/ModelTest.php

@@ -5,9 +5,9 @@
 namespace MongoDB\Laravel\Tests\Models;
 
 use MongoDB\Laravel\Eloquent\HasSchemaVersion;
-use MongoDB\Laravel\Eloquent\Model as Eloquent;
+use MongoDB\Laravel\Eloquent\Model;
 
-class SchemaVersion extends Eloquent
+class SchemaVersion extends Model
 {
     use HasSchemaVersion;
 

tests/Models/SchemaVersion.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MongoDB\Laravel\Tests\Scout\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Schema\SQLiteBuilder;
+use Illuminate\Support\Facades\Schema;
+use Laravel\Scout\Searchable;
+use MongoDB\Laravel\Eloquent\SoftDeletes;
+
+use function assert;
+
+class ScoutUser extends Model
+{
+    use Searchable;
+    use SoftDeletes;
+
+    protected $connection       = 'sqlite';
+    protected $table            = 'scout_users';
+    protected static $unguarded = true;
+
+    /**
+     * Create the SQL table for the model.
+     */
+    public static function executeSchema(): void
+    {
+        $schema = Schema::connection('sqlite');
+        assert($schema instanceof SQLiteBuilder);
+
+        $schema->dropIfExists('scout_users');
+        $schema->create('scout_users', function (Blueprint $table) {
+            $table->increments('id');
+            $table->string('name');
+            $table->string('email')->nullable();
+            $table->date('email_verified_at')->nullable();
+            $table->timestamps();
+            $table->softDeletes();
+        });
+    }
+}

tests/Scout/Models/ScoutUser.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MongoDB\Laravel\Tests\Scout\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Laravel\Scout\Searchable;
+use MongoDB\Laravel\Eloquent\DocumentModel;
+
+class SearchableInSameNamespace extends Model
+{
+    use DocumentModel;
+    use Searchable;
+
+    protected $keyType = 'string';
+    protected $connection = 'mongodb';
+    protected $fillable = ['name'];
+
+    /**
+     * Using the same collection as the model collection as Scout index
+     * is prohibited to prevent erasing the data.
+     *
+     * @see Searchable::searchableAs()
+     */
+    public function indexableAs(): string
+    {
+        return $this->getTable();
+    }
+}

tests/Scout/Models/SearchableInSameNamespace.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace MongoDB\Laravel\Tests\Scout\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
+use Laravel\Scout\Searchable;
+
+class SearchableModel extends Model
+{
+    use Searchable;
+    use SoftDeletes;
+
+    protected $connection = 'sqlite';
+    protected $fillable = ['id', 'name', 'date'];
+
+    /** @see Searchable::searchableAs() */
+    public function searchableAs(): string
+    {
+        return 'collection_searchable';
+    }
+
+    /** @see Searchable::indexableAs() */
+    public function indexableAs(): string
+    {
+        return 'collection_indexable';
+    }
+
+    /**
+     * Overriding the `getScoutKey` method to ensure the custom key is used for indexing
+     * and searching the model.
+     *
+     * @see Searchable::getScoutKey()
+     */
+    public function getScoutKey(): string
+    {
+        return $this->getAttribute($this->getScoutKeyName()) ?: 'key_' . $this->getKey();
+    }
+
+    /**
+     * This method must be overridden when the `getScoutKey` method is also overridden,
+     * to support model serialization for async indexing jobs.
+     *
+     * @see Searchable::getScoutKeyName()
+     */
+    public function getScoutKeyName(): string
+    {
+        return 'scout_key';
+    }
+}

tests/Scout/Models/SearchableModel.php

@@ -0,0 +1,262 @@
+<?php
+
+namespace MongoDB\Laravel\Tests\Scout;
+
+use DateTimeImmutable;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\LazyCollection;
+use Laravel\Scout\ScoutServiceProvider;
+use LogicException;
+use MongoDB\Laravel\Tests\Scout\Models\ScoutUser;
+use MongoDB\Laravel\Tests\Scout\Models\SearchableInSameNamespace;
+use MongoDB\Laravel\Tests\TestCase;
+use Override;
+use PHPUnit\Framework\Attributes\Depends;
+
+use function array_merge;
+use function count;
+use function env;
+use function Orchestra\Testbench\artisan;
+use function range;
+use function sprintf;
+use function usleep;
+
+class ScoutIntegrationTest extends TestCase
+{
+    #[Override]
+    protected function getPackageProviders($app): array
+    {
+        return array_merge(parent::getPackageProviders($app), [ScoutServiceProvider::class]);
+    }
+
+    #[Override]
+    protected function getEnvironmentSetUp($app): void
+    {
+        parent::getEnvironmentSetUp($app);
+
+        $app['config']->set('scout.driver', 'mongodb');
+        $app['config']->set('scout.prefix', 'prefix_');
+    }
+
+    public function setUp(): void
+    {
+        parent::setUp();
+
+        $this->skipIfSearchIndexManagementIsNotSupported();
+
+        // Init the SQL database with some objects that will be indexed
+        // Test data copied from Laravel Scout tests
+        // https://github.com/laravel/scout/blob/10.x/tests/Integration/SearchableTests.php
+        ScoutUser::executeSchema();
+
+        $collect = LazyCollection::make(function () {
+            yield ['name' => 'Laravel Framework'];
+
+            foreach (range(2, 10) as $key) {
+                yield ['name' => 'Example ' . $key];
+            }
+
+            yield ['name' => 'Larry Casper', 'email_verified_at' => null];
+            yield ['name' => 'Reta Larkin'];
+
+            foreach (range(13, 19) as $key) {
+                yield ['name' => 'Example ' . $key];
+            }
+
+            yield ['name' => 'Prof. Larry Prosacco DVM', 'email_verified_at' => null];
+
+            foreach (range(21, 38) as $key) {
+                yield ['name' => 'Example ' . $key, 'email_verified_at' => null];
+            }
+
+            yield ['name' => 'Linkwood Larkin', 'email_verified_at' => null];
+            yield ['name' => 'Otis Larson MD'];
+            yield ['name' => 'Gudrun Larkin'];
+            yield ['name' => 'Dax Larkin'];
+            yield ['name' => 'Dana Larson Sr.'];
+            yield ['name' => 'Amos Larson Sr.'];
+        });
+
+        $id = 0;
+        $date = new DateTimeImmutable('2021-01-01 00:00:00');
+        foreach ($collect as $data) {
+            $data = array_merge(['id' => ++$id, 'email_verified_at' => $date], $data);
+            ScoutUser::create($data)->save();
+        }
+
+        self::assertSame(44, ScoutUser::count());
+    }
+
+    /** This test create the search index for tests performing search */
+    public function testItCanCreateTheCollection()
+    {
+        $collection = DB::connection('mongodb')->getCollection('prefix_scout_users');
+        $collection->drop();
+
+        // Recreate the indexes using the artisan commands
+        // Ensure they return a success exit code (0)
+        self::assertSame(0, artisan($this, 'scout:delete-index', ['name' => ScoutUser::class]));
+        self::assertSame(0, artisan($this, 'scout:index', ['name' => ScoutUser::class]));
+        self::assertSame(0, artisan($this, 'scout:import', ['model' => ScoutUser::class]));
+
+        self::assertSame(44, $collection->countDocuments());
+
+        $searchIndexes = $collection->listSearchIndexes(['name' => 'scout']);
+        self::assertCount(1, $searchIndexes);
+
+        // Wait for all documents to be indexed asynchronously
+        $i = 100;
+        while (true) {
+            $indexedDocuments = $collection->aggregate([
+                ['$search' => ['index' => 'scout', 'exists' => ['path' => 'name']]],
+            ])->toArray();
+
+            if (count($indexedDocuments) >= 44) {
+                break;
+            }
+
+            if ($i-- === 0) {
+                self::fail('Documents not indexed');
+            }
+
+            usleep(100_000);
+        }
+
+        self::assertCount(44, $indexedDocuments);
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUseBasicSearch()
+    {
+        // All the search queries use "sort" option to ensure the results are deterministic
+        $results = ScoutUser::search('lar')->take(10)->orderBy('id')->get();
+
+        self::assertSame([
+            1 => 'Laravel Framework',
+            11 => 'Larry Casper',
+            12 => 'Reta Larkin',
+            20 => 'Prof. Larry Prosacco DVM',
+            39 => 'Linkwood Larkin',
+            40 => 'Otis Larson MD',
+            41 => 'Gudrun Larkin',
+            42 => 'Dax Larkin',
+            43 => 'Dana Larson Sr.',
+            44 => 'Amos Larson Sr.',
+        ], $results->pluck('name', 'id')->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUseBasicSearchCursor()
+    {
+        // All the search queries use "sort" option to ensure the results are deterministic
+        $results = ScoutUser::search('lar')->take(10)->orderBy('id')->cursor();
+
+        self::assertSame([
+            1 => 'Laravel Framework',
+            11 => 'Larry Casper',
+            12 => 'Reta Larkin',
+            20 => 'Prof. Larry Prosacco DVM',
+            39 => 'Linkwood Larkin',
+            40 => 'Otis Larson MD',
+            41 => 'Gudrun Larkin',
+            42 => 'Dax Larkin',
+            43 => 'Dana Larson Sr.',
+            44 => 'Amos Larson Sr.',
+        ], $results->pluck('name', 'id')->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUseBasicSearchWithQueryCallback()
+    {
+        $results = ScoutUser::search('lar')->take(10)->orderBy('id')->query(function ($query) {
+            return $query->whereNotNull('email_verified_at');
+        })->get();
+
+        self::assertSame([
+            1 => 'Laravel Framework',
+            12 => 'Reta Larkin',
+            40 => 'Otis Larson MD',
+            41 => 'Gudrun Larkin',
+            42 => 'Dax Larkin',
+            43 => 'Dana Larson Sr.',
+            44 => 'Amos Larson Sr.',
+        ], $results->pluck('name', 'id')->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUseBasicSearchToFetchKeys()
+    {
+        $results = ScoutUser::search('lar')->orderBy('id')->take(10)->keys();
+
+        self::assertSame([1, 11, 12, 20, 39, 40, 41, 42, 43, 44], $results->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUseBasicSearchWithQueryCallbackToFetchKeys()
+    {
+        $results = ScoutUser::search('lar')->take(10)->orderBy('id', 'desc')->query(function ($query) {
+            return $query->whereNotNull('email_verified_at');
+        })->keys();
+
+        self::assertSame([44, 43, 42, 41, 40, 39, 20, 12, 11, 1], $results->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUsePaginatedSearch()
+    {
+        $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 1);
+        $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 2);
+
+        self::assertSame([
+            1 => 'Laravel Framework',
+            11 => 'Larry Casper',
+            12 => 'Reta Larkin',
+            20 => 'Prof. Larry Prosacco DVM',
+            39 => 'Linkwood Larkin',
+        ], $page1->pluck('name', 'id')->all());
+
+        self::assertSame([
+            40 => 'Otis Larson MD',
+            41 => 'Gudrun Larkin',
+            42 => 'Dax Larkin',
+            43 => 'Dana Larson Sr.',
+            44 => 'Amos Larson Sr.',
+        ], $page2->pluck('name', 'id')->all());
+    }
+
+    #[Depends('testItCanCreateTheCollection')]
+    public function testItCanUsePaginatedSearchWithQueryCallback()
+    {
+        $queryCallback = function ($query) {
+            return $query->whereNotNull('email_verified_at');
+        };
+
+        $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 1);
+        $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 2);
+
+        self::assertSame([
+            1 => 'Laravel Framework',
+            12 => 'Reta Larkin',
+        ], $page1->pluck('name', 'id')->all());
+
+        self::assertSame([
+            40 => 'Otis Larson MD',
+            41 => 'Gudrun Larkin',
+            42 => 'Dax Larkin',
+            43 => 'Dana Larson Sr.',
+            44 => 'Amos Larson Sr.',
+        ], $page2->pluck('name', 'id')->all());
+    }
+
+    public function testItCannotIndexInTheSameNamespace()
+    {
+        self::expectException(LogicException::class);
+        self::expectExceptionMessage(sprintf(
+            'The MongoDB Scout collection "%s.searchable_in_same_namespaces" must use a different collection from the collection name of the model "%s". Set the "scout.prefix" configuration or use a distinct MongoDB database',
+            env('MONGODB_DATABASE', 'unittest'),
+            SearchableInSameNamespace::class,
+        ),);
+
+        SearchableInSameNamespace::create(['name' => 'test']);
+    }
+}