Fix Typesense pagination issue when using query callback

config/scout.php

@@ -173,6 +173,7 @@
             'num_retries' => env('TYPESENSE_NUM_RETRIES', 3),
             'retry_interval_seconds' => env('TYPESENSE_RETRY_INTERVAL_SECONDS', 1),
         ],
+        // 'max_total_results' => env('TYPESENSE_MAX_TOTAL_RESULTS', 1000),
         'model-settings' => [
             // User::class => [
             //     'collection-schema' => [

src/EngineManager.php

@@ -153,9 +153,10 @@ protected function ensureMeilisearchClientIsInstalled()
      */
     public function createTypesenseDriver()
     {
+        $config = config('scout.typesense');
         $this->ensureTypesenseClientIsInstalled();
 
-        return new TypesenseEngine(new Typesense(config('scout.typesense.client-settings')));
+        return new TypesenseEngine(new Typesense($config['client-settings']), $config['max_total_results'] ?? 1000);
     }
 
     /**

src/Engines/TypesenseEngine.php

@@ -29,14 +29,29 @@ class TypesenseEngine extends Engine
      */
     protected array $searchParameters = [];
 
+    /**
+     * The maximum number of results that can be fetched per page.
+     *
+     * @var int
+     */
+    private int $maxPerPage = 250;
+
+    /**
+     * The maximum number of results that can be fetched during pagination.
+     *
+     * @var int
+     */
+    protected int $maxTotalResults;
+
     /**
      * Create new Typesense engine instance.
      *
      * @param  Typesense  $typesense
      */
-    public function __construct(Typesense $typesense)
+    public function __construct(Typesense $typesense, int $maxTotalResults)
     {
         $this->typesense = $typesense;
+        $this->maxTotalResults = $maxTotalResults;
     }
 
     /**
@@ -186,9 +201,14 @@ protected function deleteDocument(TypesenseCollection $collectionIndex, $modelId
      */
     public function search(Builder $builder)
     {
+        // If the limit exceeds Typesense's capabilities, perform a paginated search...
+        if ($builder->limit >= $this->maxPerPage) {
+            return $this->performPaginatedSearch($builder);
+        }
+
         return $this->performSearch(
             $builder,
-            $this->buildSearchParameters($builder, 1, $builder->limit)
+            $this->buildSearchParameters($builder, 1, $builder->limit ?? $this->maxPerPage)
         );
     }
 
@@ -232,6 +252,52 @@ protected function performSearch(Builder $builder, array $options = []): mixed
         return $documents->search($options);
     }
 
+    /**
+     * Perform a paginated search on the engine.
+     *
+     * @param  \Laravel\Scout\Builder  $builder
+     * @return mixed
+     *
+     * @throws \Http\Client\Exception
+     * @throws \Typesense\Exceptions\TypesenseClientError
+     */
+    protected function performPaginatedSearch(Builder $builder)
+    {
+        $page = 1;
+        $limit = min($builder->limit ?? $this->maxPerPage, $this->maxPerPage, $this->maxTotalResults);
+        $remainingResults = min($builder->limit ?? $this->maxTotalResults, $this->maxTotalResults);
+
+        $results = new Collection;
+
+        while ($remainingResults > 0) {
+            $searchResults = $this->performSearch(
+                $builder,
+                $this->buildSearchParameters($builder, $page, $limit)
+            );
+
+            $results = $results->concat($searchResults['hits'] ?? []);
+
+            if ($page === 1) {
+                $totalFound = $searchResults['found'] ?? 0;
+            }
+
+            $remainingResults -= $limit;
+            $page++;
+
+            if (count($searchResults['hits'] ?? []) < $limit) {
+                break;
+            }
+        }
+
+        return [
+            'hits' => $results->all(),
+            'found' => $results->count(),
+            'out_of' => $totalFound,
+            'page' => 1,
+            'request_params' => $this->buildSearchParameters($builder, 1, $builder->limit ?? $this->maxPerPage),
+        ];
+    }
+
     /**
      * Build the search parameters for a given Scout query builder.
      *

tests/Integration/SearchableTests.php

@@ -104,4 +104,12 @@ protected function itCanUsePaginatedSearchWithQueryCallback()
             User::search('lar')->take(10)->query($queryCallback)->paginate(5, 'page', 2),
         ];
     }
+
+    protected function itCanUsePaginatedSearchWithEmptyQueryCallback()
+    {
+        $queryCallback = function ($query) {
+        };
+
+        return User::search('*')->query($queryCallback)->paginate();
+    }
 }

tests/Integration/TypesenseSearchableTest.php

@@ -164,6 +164,14 @@ public function test_it_can_use_paginated_search_with_query_callback()
         ], $page2->pluck('name', 'id')->all());
     }
 
+    public function test_it_can_usePaginatedSearchWithEmptyQueryCallback()
+    {
+        $res = $this->itCanUsePaginatedSearchWithEmptyQueryCallback();
+
+        $this->assertSame($res->total(), 44);
+        $this->assertSame($res->lastPage(), 3);
+    }
+
     protected static function scoutDriver(): string
     {
         return 'typesense';

tests/Unit/TypesenseEngineTest.php

@@ -27,7 +27,7 @@ protected function setUp(): void
         // Mock the Typesense client and pass it to the engine constructor
         $typesenseClient = $this->createMock(TypesenseClient::class);
         $this->engine = $this->getMockBuilder(TypesenseEngine::class)
-            ->setConstructorArgs([$typesenseClient])
+            ->setConstructorArgs([$typesenseClient, 1000])
             ->onlyMethods(['getOrCreateCollectionFromModel', 'buildSearchParameters'])
             ->getMock();
     }