allow "_id" filters in index queries (#23979)

allow indexes to be queried with filters on `"_id"`, including the `by_id` index.

also adds typescript support for by_id and for `_id` in every index

this enables manual pagination (not easily, since you have to do multiple queries, but at least now it's possible) and also makes `db.get` just a special case of `db.query`, which helps with parallelization efforts.

added tests.

GitOrigin-RevId: 779e62021d30cd9d04386e141f657940eeb2b69c

Author: ldanilek

crates/common/src/bootstrap_model/index/database_index/indexed_fields.rs

@@ -2,6 +2,7 @@ use std::{
     collections::HashSet,
     convert::TryFrom,
     fmt::Display,
+    iter,
     ops::Deref,
 };
 
@@ -44,6 +45,10 @@ impl IndexedFields {
             .expect("Invalid _creationTime field path");
         IndexedFields(vec![field_path].into())
     }
+
+    pub fn iter_with_id(&self) -> impl Iterator<Item = &FieldPath> {
+        self.iter().chain(iter::once(&*ID_FIELD_PATH))
+    }
 }
 
 impl HeapSize for IndexedFields {

crates/common/src/query.rs

@@ -189,13 +189,13 @@ impl IndexRange {
         // NB: `indexed_fields` does not include the implicit `_id` field at the end of
         // every index, so this omission prevents the user from using it in an
         // index expression.
-        let index_rank: BTreeMap<_, _> = indexed_fields[..]
-            .iter()
+        let index_rank: BTreeMap<_, _> = indexed_fields
+            .iter_with_id()
             .enumerate()
             .map(|(i, field_name)| (field_name, i))
             .collect();
         anyhow::ensure!(
-            index_rank.len() == indexed_fields.len(),
+            index_rank.len() == indexed_fields.len() + 1,
             "{index_name} has duplicate fields?"
         );
 
@@ -237,7 +237,7 @@ impl IndexRange {
 
         let query_fields = QueryFields(used_paths.clone());
 
-        let mut fields_iter = indexed_fields.iter();
+        let mut fields_iter = indexed_fields.iter_with_id();
         for field_path in used_paths {
             let matching_field = fields_iter.next().ok_or_else(|| {
                 invalid_index_range(&index_name, &indexed_fields, &query_fields, &field_path)

crates/isolate/src/tests/basic.rs

@@ -162,6 +162,11 @@ async fn test_insert_and_get(rt: TestRuntime) -> anyhow::Result<()> {
     assert_eq!(id2, id);
     must_let!(let Some(field) = obj.get("field"));
     assert_eq!(field, &value);
+
+    // Get the object by index lookup.
+    must_let!(let ConvexValue::Object(obj) = t.query("basic:getObjectById", assert_obj!("id" => id.clone())).await?);
+    must_let!(let Some(id3) = obj.get("_id"));
+    assert_eq!(id3, id);
     Ok(())
 }
 

npm-packages/convex/src/server/index.ts

@@ -128,7 +128,7 @@ export type {
   WithoutSystemFields,
   WithOptionalSystemFields,
   SystemIndexes,
-  IndexTiebreakerField,
+  IndexTiebreakerFields,
 } from "./system_fields.js";
 export { httpRouter, HttpRouter, ROUTABLE_HTTP_METHODS } from "./router.js";
 export type { RoutableMethod } from "./router.js";

npm-packages/convex/src/server/schema.test.ts

@@ -432,11 +432,17 @@ describe("DataModelFromSchemaDefinition", () => {
       | "property1"
       | "property2";
     type ExpectedIndexes = {
-      by_property1: ["property1", "_creationTime"];
-      by_property1_property2: ["property1", "property2", "_creationTime"];
+      by_property1: ["property1", "_creationTime", "_id"];
+      by_property1_property2: [
+        "property1",
+        "property2",
+        "_creationTime",
+        "_id",
+      ];
 
-      // System index
-      by_creation_time: ["_creationTime"];
+      // System indexes
+      by_creation_time: ["_creationTime", "_id"];
+      by_id: ["_id"];
     };
     type ExpectedDataModel = {
       table: {

npm-packages/convex/src/server/schema.ts

@@ -38,7 +38,7 @@ import {
 } from "../server/data_model.js";
 import {
   IdField,
-  IndexTiebreakerField,
+  IndexTiebreakerFields,
   SystemFields,
   SystemIndexes,
 } from "../server/system_fields.js";
@@ -200,7 +200,7 @@ export class TableDefinition<
       Indexes &
         Record<
           IndexName,
-          [FirstFieldPath, ...RestFieldPaths, IndexTiebreakerField]
+          [FirstFieldPath, ...RestFieldPaths, ...IndexTiebreakerFields]
         >
     >,
     SearchIndexes,

npm-packages/convex/src/server/system_fields.ts

@@ -46,16 +46,15 @@ export type WithOptionalSystemFields<Document extends GenericDocument> = Expand<
  * @public
  */
 export type SystemIndexes = {
-  // We have a system index `by_id` but developers should never have a use
-  // for querying it (`db.get(id)` is always simpler).
-  // by_id: ["_id"];
+  // Note `db.get(id)` is simpler and equivalent to a query on `by_id`.
+  // Unless the query is being built dynamically, or doing manual pagination.
+  by_id: ["_id"];
 
-  by_creation_time: ["_creationTime"];
+  by_creation_time: ["_creationTime", "_id"];
 };
 
 /**
- * Convex automatically appends "_creationTime" to the end of every index to
- * break ties if all of the other fields are identical.
+ * Convex automatically appends "_creationTime" and "_id" to the end of every index to break ties if all other fields are identical.
  * @public
  */
-export type IndexTiebreakerField = "_creationTime";
+export type IndexTiebreakerFields = ["_creationTime", "_id"];

npm-packages/udf-tests/convex/basic.ts

@@ -22,6 +22,15 @@ export const getObject = query(async ({ db }, { id }: { id: Id<any> }) => {
   return db.get(id);
 });
 
+export const getObjectById = query(
+  async ({ db }, { id }: { id: Id<"objects"> }) => {
+    return db
+      .query("objects")
+      .withIndex("by_id", (q) => q.eq("_id", id))
+      .unique();
+  },
+);
+
 export const insertObject = mutation(async ({ db }, obj) => {
   const id = await db.insert("objects", obj);
   return await db.get(id);