diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 49a33e403..37772e1b2 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -666,10 +666,12 @@ abstract public function renameIndex(string $collection, string $old, string $ne * @param array $lengths * @param array $orders * @param array $indexAttributeTypes + * @param array $collation + * @param int $ttl * * @return bool */ - abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool; + abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool; /** * Delete Index @@ -1483,4 +1485,14 @@ public function getSupportForRegex(): bool { return $this->getSupportForPCRERegex() || $this->getSupportForPOSIXRegex(); } + + /** + * Are ttl indexes supported? + * + * @return bool + */ + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b8110b039..242b0d9ad 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -715,7 +715,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @return bool * @throws DatabaseException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $metadataCollection = new Document(['$id' => Database::METADATA]); $collection = $this->getDocument($metadataCollection, $collection); @@ -2255,4 +2255,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fdcdd80c2..68935742d 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in $orders = $index->getAttribute('orders'); // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->sharedTables && $index->getAttribute('type') !== Database::INDEX_TTL) { $key['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -508,6 +508,9 @@ public function createCollection(string $name, array $attributes = [], array $in $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); $unique = true; break; + case Database::INDEX_TTL: + $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); + break; default: // index not supported return false; @@ -526,6 +529,14 @@ public function createCollection(string $name, array $attributes = [], array $in $newIndexes[$i]['default_language'] = 'none'; } + // Handle TTL indexes + if ($index->getAttribute('type') === Database::INDEX_TTL) { + $ttl = $index->getAttribute('ttl', 0); + if ($ttl > 0) { + $newIndexes[$i]['expireAfterSeconds'] = $ttl; + } + } + // Add partial filter for indexes to avoid indexing null values if (in_array($index->getAttribute('type'), [ Database::INDEX_UNIQUE, @@ -901,10 +912,11 @@ public function deleteRelationship( * @param array $orders * @param array $indexAttributeTypes * @param array $collation + * @param int $ttl * @return bool * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); $id = $this->filter($id); @@ -913,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['name'] = $id; // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->sharedTables && $type !== Database::INDEX_TTL) { $indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -933,6 +945,8 @@ public function createIndex(string $collection, string $id, string $type, array case Database::INDEX_UNIQUE: $indexes['unique'] = true; break; + case Database::INDEX_TTL: + break; default: return false; } @@ -961,6 +975,11 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['default_language'] = 'none'; } + // Handle TTL indexes + if ($type === Database::INDEX_TTL && $ttl > 0) { + $indexes['expireAfterSeconds'] = $ttl; + } + // Add partial filter for indexes to avoid indexing null values if (in_array($type, [Database::INDEX_UNIQUE, Database::INDEX_KEY])) { $partialFilter = []; @@ -1073,7 +1092,7 @@ public function renameIndex(string $collection, string $old, string $new): bool try { $deletedindex = $this->deleteIndex($collection, $old); - $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes); + $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, [], $index['ttl'] ?? 0); } catch (\Exception $e) { throw $this->processException($e); } @@ -3395,4 +3414,9 @@ public function getSupportForTrigramIndex(): bool { return false; } + + public function getSupportTTLIndexes(): bool + { + return true; + } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 308013738..d5740cf66 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -317,4 +317,9 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope // For all other operators, use parent implementation return parent::getOperatorSQL($column, $operator, $bindIndex); } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index d70a836ea..1d8c892e6 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -210,7 +210,7 @@ public function renameIndex(string $collection, string $old, string $new): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } @@ -649,4 +649,9 @@ public function getSupportForAlterLocks(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + + public function getSupportTTLIndexes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 050180a0a..5cfe39d55 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -319,6 +319,7 @@ public function createCollection(string $name, array $attributes = [], array $in } } $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); if ($indexType === Database::INDEX_SPATIAL && count($indexOrders)) { throw new DatabaseException('Spatial indexes with explicit orders are not supported. Remove the orders to create this index.'); } @@ -329,7 +330,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes, [], $indexOrders, - $indexAttributesWithType + $indexAttributesWithType, + [], + $indexTtl ); } } catch (PDOException $e) { @@ -876,7 +879,7 @@ public function deleteRelationship( * @return bool */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $collection = $this->filter($collection); $id = $this->filter($id); @@ -2829,4 +2832,9 @@ protected function getSQLTable(string $name): string return "{$this->quote($this->getDatabase())}.{$this->quote($table)}"; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 948070654..55e195dc2 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -216,8 +216,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes = $index->getAttribute('attributes', []); $indexLengths = $index->getAttribute('lengths', []); $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); - $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders); + $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders, [], [], $indexTtl); } $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []); @@ -455,7 +456,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws Exception * @throws PDOException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $name = $this->filter($collection); $id = $this->filter($id); @@ -1908,4 +1909,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index a2bc2da55..65526c348 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -86,6 +86,7 @@ class Database public const INDEX_HNSW_COSINE = 'hnsw_cosine'; public const INDEX_HNSW_DOT = 'hnsw_dot'; public const INDEX_TRIGRAM = 'trigram'; + public const INDEX_TTL = 'ttl'; // Max limits public const MAX_INT = 2147483647; @@ -1648,6 +1649,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); foreach ($indexes as $index) { if (!$validator->isValid($index)) { @@ -2798,6 +2800,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); foreach ($indexes as $index) { @@ -3603,6 +3606,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @param array $attributes * @param array $lengths * @param array $orders + * @param int $ttl * * @return bool * @throws AuthorizationException @@ -3613,14 +3617,13 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws StructureException * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool { if (empty($attributes)) { throw new DatabaseException('Missing attributes'); } $collection = $this->silent(fn () => $this->getCollection($collection)); - // index IDs are case-insensitive $indexes = $collection->getAttribute('indexes', []); @@ -3671,6 +3674,7 @@ public function createIndex(string $collection, string $id, string $type, array 'attributes' => $attributes, 'lengths' => $lengths, 'orders' => $orders, + 'ttl' => $ttl ]); if ($this->validate) { @@ -3693,6 +3697,7 @@ public function createIndex(string $collection, string $id, string $type, array $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); if (!$validator->isValid($index)) { throw new IndexException($validator->getDescription()); @@ -3702,7 +3707,7 @@ public function createIndex(string $collection, string $id, string $type, array $collection->setAttribute('indexes', $index, Document::SET_TYPE_APPEND); try { - $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes); + $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes, [], $ttl); if (!$created) { throw new DatabaseException('Failed to create index'); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index d9a6d09df..3c65a88a9 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -469,9 +469,9 @@ public function deleteAttribute(string $collection, string $id): bool return $result; } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool { - $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders); + $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders, $ttl); if ($this->destination === null) { return $result; @@ -502,7 +502,8 @@ public function createIndex(string $collection, string $id, string $type, array $document->getAttribute('type'), $document->getAttribute('attributes'), $document->getAttribute('lengths'), - $document->getAttribute('orders') + $document->getAttribute('orders'), + $document->getAttribute('ttl', 0) ); } catch (\Throwable $err) { $this->logError('createIndex', $err); diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index e2fc70a0b..1e367d1ad 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -54,6 +54,7 @@ public function __construct( protected bool $supportForKeyIndexes = true, protected bool $supportForUniqueIndexes = true, protected bool $supportForFulltextIndexes = true, + protected bool $supportForTTLIndexes = false, ) { foreach ($attributes as $attribute) { $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); @@ -156,6 +157,9 @@ public function isValid($value): bool if (!$this->checkKeyUniqueFulltextSupport($value)) { return false; } + if (!$this->checkTTLIndexes($value)) { + return false; + } return true; } @@ -222,8 +226,15 @@ public function checkValidIndex(Document $index): bool } break; + case Database::INDEX_TTL: + if (!$this->supportForTTLIndexes) { + $this->message = 'TTL indexes are not supported'; + return false; + } + break; + default: - $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM; + $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM . ', '.Database::INDEX_TTL; return false; } return true; @@ -729,4 +740,57 @@ public function checkObjectIndexes(Document $index): bool return true; } + + public function checkTTLIndexes(Document $index): bool + { + $type = $index->getAttribute('type'); + + $attributes = $index->getAttribute('attributes', []); + $orders = $index->getAttribute('orders', []); + $ttl = $index->getAttribute('ttl', 0); + if ($type !== Database::INDEX_TTL) { + return true; + } + + if (count($attributes) !== 1) { + $this->message = 'TTL index can be created on a single datetime attribute'; + return false; + } + + if (empty($orders)) { + $this->message = 'TTL index need explicit orders. Add the orders to create this index.'; + return false; + } + + $attributeName = $attributes[0] ?? ''; + $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); + $attributeType = $attribute->getAttribute('type', ''); + + if ($this->supportForAttributes && $attributeType !== Database::VAR_DATETIME) { + $this->message = 'TTL index can only be created on datetime attributes. Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; + return false; + } + + if ($ttl <= 0) { + $this->message = 'TTL must be at least 1 second'; + return false; + } + + foreach ($this->indexes as $existingIndex) { + $existingAttributes = $existingIndex->getAttribute('attributes', []); + $existingOrders = $existingIndex->getAttribute('orders', []); + $existingType = $existingIndex->getAttribute('type', ''); + if ($this->supportForAttributes && $existingType !== Database::INDEX_TTL) { + continue; + } + $attributeAlreadyPresent = ($this->supportForAttributes && in_array($attribute->getId(), $existingAttributes)) || in_array($attributeName, $existingAttributes); + $ordersMatched = empty(array_diff($existingOrders, $orders)); + if ($attributeAlreadyPresent && $ordersMatched) { + $this->message = 'There is already an index with the same attributes and orders'; + return false; + } + } + + return true; + } } diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index e5eda16d0..6c5e21aaa 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -784,4 +784,265 @@ public function testTrigramIndexValidation(): void $database->deleteCollection($collectionId); } } + + public function testTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $database->createDocuments($col, [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + ]) + ]); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires_new', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_expires_new', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + // raised in the mongo level + $this->assertStringContainsString('Index already exists', $e->getMessage()); + } + + // Cleanup + $database->deleteCollection($col); + } } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 856d08263..079a1c7c3 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -6,6 +6,7 @@ use Throwable; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; @@ -1865,4 +1866,266 @@ public function testSchemalessNestedObjectAttributeQueries(): void $database->deleteCollection($col); } + + public function testSchemalessTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + 'data' => 'will expire in 2 hours' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + 'data' => 'will expire in 1 hour' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + 'data' => 'already expired' + ])); + + // Verify documents were created + $this->assertEquals('doc1', $doc1->getId()); + $this->assertEquals('doc2', $doc2->getId()); + $this->assertEquals('doc3', $doc3->getId()); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testSchemalessTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires_new', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_expires_new', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('Index already exists', $e->getMessage()); + } + + $database->deleteCollection($col); + } } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 5dfe80e4e..664874051 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -596,4 +596,183 @@ public function testTrigramIndexValidation(): void $this->assertFalse($validatorNoSupport->isValid($validIndex)); $this->assertEquals('Trigram indexes are not supported', $validatorNoSupport->getDescription()); } + + /** + * @throws Exception + */ + public function testTTLIndexValidation(): void + { + $collection = new Document([ + '$id' => ID::custom('test'), + 'name' => 'test', + 'attributes' => [ + new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]), + new Document([ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), + ], + 'indexes' => [] + ]); + + // Validator with supportForTTLIndexes enabled + $validator = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + // Valid: TTL index on single datetime attribute with valid TTL + $validIndex = new Document([ + '$id' => ID::custom('idx_ttl_valid'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertTrue($validator->isValid($validIndex)); + + // Invalid: TTL index with TTL = 0 + $invalidIndexZero = new Document([ + '$id' => ID::custom('idx_ttl_zero'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 0, + ]); + $this->assertFalse($validator->isValid($invalidIndexZero)); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); + + // Invalid: TTL index with TTL < 0 + $invalidIndexNegative = new Document([ + '$id' => ID::custom('idx_ttl_negative'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => -100, + ]); + $this->assertFalse($validator->isValid($invalidIndexNegative)); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); + + // Invalid: TTL index on non-datetime attribute + $invalidIndexType = new Document([ + '$id' => ID::custom('idx_ttl_invalid_type'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexType)); + $this->assertStringContainsString('TTL index can only be created on datetime attributes', $validator->getDescription()); + + // Invalid: TTL index on multiple attributes + $invalidIndexMulti = new Document([ + '$id' => ID::custom('idx_ttl_multi'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt', 'name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexMulti)); + $this->assertStringContainsString('TTL index can be created on a single datetime attribute', $validator->getDescription()); + + // Invalid: TTL index without orders + $invalidIndexNoOrders = new Document([ + '$id' => ID::custom('idx_ttl_no_orders'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexNoOrders)); + $this->assertEquals('TTL index need explicit orders. Add the orders to create this index.', $validator->getDescription()); + + // Valid: TTL index with minimum valid TTL (1 second) + $validIndexMin = new Document([ + '$id' => ID::custom('idx_ttl_min'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 1, + ]); + $this->assertTrue($validator->isValid($validIndexMin)); + + // Invalid: TTL index on same attribute when another TTL index already exists + $collection->setAttribute('indexes', $validIndex, Document::SET_TYPE_APPEND); + $validatorWithExisting = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + $duplicateTTLIndex = new Document([ + '$id' => ID::custom('idx_ttl_duplicate'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200, + ]); + $this->assertFalse($validatorWithExisting->isValid($duplicateTTLIndex)); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $validatorWithExisting->getDescription()); + + // Validator with supportForTrigramIndexes disabled should reject TTL + $validatorNoSupport = new Index($collection->getAttribute('attributes'), $collection->getAttribute('indexes', []), 768, [], false, false, false, false, false, false, false, false, false); + $this->assertFalse($validatorNoSupport->isValid($validIndex)); + $this->assertEquals('TTL indexes are not supported', $validatorNoSupport->getDescription()); + } }