diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78acdcd..117b322 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,19 +16,21 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Node setup - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 with: - node-version: "20.19.5" + cache-dependency-path: package.json + node-version: "20.x" + cache: "npm" - - name: Install dependencies + - name: Install and build run: | npm i npm run build - - name: pkg-pr-new - run: npx pkg-pr-new publish . + - name: Publish package for testing branch + run: npx pkg-pr-new publish || echo "Have you set up pkg-pr-new for this repo?" - name: Test run: | npm run test diff --git a/.prettierrc.json b/.prettierrc.json index 757fd64..f3877f7 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -1,3 +1,4 @@ { - "trailingComma": "es5" + "trailingComma": "all", + "proseWrap": "always" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff55c4..c94b77e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.6.0 + +- Adds /test and /\_generated/component.js entrypoints +- Drops commonjs support +- Improves source mapping for generated files +- Changes to a statically generated component API + ## 0.5.4 - Support gateway string IDs for models @@ -28,18 +35,17 @@ - ai is a regular dependency - namespaces can be deleted if there are no entries left in them -- namespaces can be synchronously deleted from an action if there - are entries in them +- namespaces can be synchronously deleted from an action if there are entries in + them ## 0.3.3 - Allow deleting an entry by key asynchronously or sync -- Deprecated: `.delete` from mutations is deprecated. - `.delete` is now synchronous for an entry. - Use `.deleteAsync` from mutations instead. +- Deprecated: `.delete` from mutations is deprecated. `.delete` is now + synchronous for an entry. Use `.deleteAsync` from mutations instead. - Fix: Delete embeddings when deleting entry -- Fix: Replacing small documents by key no longer leaves - them in "pending" state. +- Fix: Replacing small documents by key no longer leaves them in "pending" + state. ## 0.3.2 @@ -57,7 +63,8 @@ - Demote document titles to h2 when auto-generating prompt template - Rename replacedVersion -> replaced{Entry,Namespace} to match onComplete -- Allow listing documents by status without specifying a namespace (e.g. vacuuming) +- Allow listing documents by status without specifying a namespace (e.g. + vacuuming) - Return replacedAt when listing documents ## 0.1.7/0.3.0 @@ -65,7 +72,8 @@ - Renamed to RAG - Adds a default chunker, so you can pass `text` to `add[Async]` - Adds a `generateText` with default prompt formatting for one-off generation. -- OnComplete handler now has updated status for the replaced & new entry/namespace +- OnComplete handler now has updated status for the replaced & new + entry/namespace - Example showcases prompting as well as searching. ## 0.1.6 @@ -79,8 +87,8 @@ ## 0.1.4 - Allow adding files asynchronously -- Allow passing an onComplete handler to creating entries - or namespaces, that is called when they are no longer pending. +- Allow passing an onComplete handler to creating entries or namespaces, that is + called when they are no longer pending. - Support generic type-safe metadata to be stored on the entry. - Updated the example to also show uploading files via http. @@ -89,8 +97,8 @@ - Renamed doc to entry - Allows passing vectorScoreThreshold to search - More convenient `text` returned from search -- Enables passing in your own embedding parameter to add - -> Allows adding (a few chunks) from a mutation. +- Enables passing in your own embedding parameter to add -> Allows adding (a few + chunks) from a mutation. ## 0.1.2 @@ -99,8 +107,8 @@ ## 0.1.1 - Vector search over chunked content, with namespaces, search filters, etc. -- You can also gracefully transition between models, embedding lengths, - chunking strategies, and versions, with automatically versioned namespaces. +- You can also gracefully transition between models, embedding lengths, chunking + strategies, and versions, with automatically versioned namespaces. - See the example for injesting pdfs, images, audio, and text! - List namespaces by status, entries by namespace/status, and chunks by entry - Find older versions by content hash to restore. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8261696..416a050 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ npm run test ```sh npm run clean -npm run build +npm ci npm pack ``` @@ -34,9 +34,7 @@ npm pack npm run release ``` -#### Alpha release - -The same as above, but it will publish the release with the `@alpha` tag: +or for alpha release: ```sh npm run alpha diff --git a/README.md b/README.md index 8057a0d..5c65bcd 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ -A component for semantic search, usually used to look up context for LLMs. -Use with an Agent for Retrieval-Augmented Generation (RAG). +A component for semantic search, usually used to look up context for LLMs. Use +with an Agent for Retrieval-Augmented Generation (RAG). [![Use AI to search HUGE amounts of text with the RAG Component](https://thumbs.video-to-markdown.com/1ff18153.jpg)](https://youtu.be/dGmtAmdAaFs) @@ -17,17 +17,20 @@ Use with an Agent for Retrieval-Augmented Generation (RAG). - **Custom Filtering**: Filter content with custom indexed fields. - **Importance Weighting**: Weight content by providing a 0 to 1 "importance". - **Chunk Context**: Get surrounding chunks for better context. -- **Graceful Migrations**: Migrate content or whole namespaces without disruption. +- **Graceful Migrations**: Migrate content or whole namespaces without + disruption. -Found a bug? Feature request? [File it here](https://github.com/get-convex/rag/issues). +Found a bug? Feature request? +[File it here](https://github.com/get-convex/rag/issues). ## Pre-requisite: Convex -You'll need an existing Convex project to use the component. -Convex is a hosted backend platform, including a database, serverless functions, -and a ton more you can learn about [here](https://docs.convex.dev/get-started). +You'll need an existing Convex project to use the component. Convex is a hosted +backend platform, including a database, serverless functions, and a ton more you +can learn about [here](https://docs.convex.dev/get-started). -Run `npm create convex` or follow any of the [quickstarts](https://docs.convex.dev/home) to set one up. +Run `npm create convex` or follow any of the +[quickstarts](https://docs.convex.dev/home) to set one up. ## Installation @@ -37,12 +40,13 @@ Install the component package: npm install @convex-dev/rag ``` -Create a `convex.config.ts` file in your app's `convex/` folder and install the component by calling `use`: +Create a `convex.config.ts` file in your app's `convex/` folder and install the +component by calling `use`: ```ts // convex/convex.config.ts import { defineApp } from "convex/server"; -import rag from "@convex-dev/rag/convex.config"; +import rag from "@convex-dev/rag/convex.config.js"; const app = defineApp(); app.use(rag); @@ -67,8 +71,8 @@ const rag = new RAG(components.rag, { ## Add context to RAG -Add content with text chunks. Each call to `add` will create a new **entry**. -It will embed the chunks automatically if you don't provide them. +Add content with text chunks. Each call to `add` will create a new **entry**. It +will embed the chunks automatically if you don't provide them. ```ts export const add = action({ @@ -83,19 +87,21 @@ export const add = action({ }); ``` -See below for how to chunk the text yourself or add content asynchronously, e.g. to handle large files. +See below for how to chunk the text yourself or add content asynchronously, e.g. +to handle large files. ## Semantic Search Search across content with vector similarity -- `text` is a string with the full content of the results, for convenience. - It is in order of the entries, with titles at each entry boundary, and - separators between non-sequential chunks. See below for more details. +- `text` is a string with the full content of the results, for convenience. It + is in order of the entries, with titles at each entry boundary, and separators + between non-sequential chunks. See below for more details. - `results` is an array of matching chunks with scores and more metadata. -- `entries` is an array of the entries that matched the query. - Each result has a `entryId` referencing one of these source entries. -- `usage` contains embedding token usage information. Will be `{ tokens: 0 }` if no embedding was performed (e.g. when passing pre-computed embeddings). +- `entries` is an array of the entries that matched the query. Each result has a + `entryId` referencing one of these source entries. +- `usage` contains embedding token usage information. Will be `{ tokens: 0 }` if + no embedding was performed (e.g. when passing pre-computed embeddings). ```ts export const search = action({ @@ -119,17 +125,17 @@ export const search = action({ Once you have searched for the context, you can use it with an LLM. -Generally you'll already be using something to make LLM requests, e.g. -the [Agent Component](https://www.convex.dev/components/agent), -which tracks the message history for you. -See the [Agent Component docs](https://docs.convex.dev/agents) -for more details on doing RAG with the Agent Component. +Generally you'll already be using something to make LLM requests, e.g. the +[Agent Component](https://www.convex.dev/components/agent), which tracks the +message history for you. See the +[Agent Component docs](https://docs.convex.dev/agents) for more details on doing +RAG with the Agent Component. However, if you just want a one-off response, you can use the `generateText` function as a convenience. -This will automatically search for relevant entries and use them as context -for the LLM, using default formatting. +This will automatically search for relevant entries and use them as context for +the LLM, using default formatting. The arguments to `generateText` are compatible with all arguments to `generateText` from the AI SDK. @@ -154,9 +160,9 @@ Note: You can specify any of the search options available on `rag.search`. ## Filtered Search -You can provide filters when adding content and use them to search. -To do this, you'll need to give the RAG component a list of the filter names. -You can optionally provide a type parameter for type safety (no runtime validation). +You can provide filters when adding content and use them to search. To do this, +you'll need to give the RAG component a list of the filter names. You can +optionally provide a type parameter for type safety (no runtime validation). Note: these filters can be OR'd together when searching. In order to get an AND, you provide a filter with a more complex value, such as `categoryAndType` below. @@ -227,14 +233,13 @@ export const searchForNewsOrSports = action({ ### Add surrounding chunks to results for context -Instead of getting just the single matching chunk, you can request -surrounding chunks so there's more context to the result. +Instead of getting just the single matching chunk, you can request surrounding +chunks so there's more context to the result. Note: If there are results that have overlapping ranges, it will not return -duplicate chunks, but instead give priority to adding the "before" context -to each chunk. -For example if you requested 2 before and 1 after, and your results were for -the same entryId indexes 1, 4, and 7, the results would be: +duplicate chunks, but instead give priority to adding the "before" context to +each chunk. For example if you requested 2 before and 1 after, and your results +were for the same entryId indexes 1, 4, and 7, the results would be: ```ts [ @@ -268,10 +273,10 @@ export const searchWithContext = action({ ## Formatting results -Formatting the results for use in a prompt depends a bit on the use case. -By default, the results will be sorted by score, not necessarily in the order -they appear in the original text. You may want to sort them by the order they -appear in the original text so they follow the flow of the original document. +Formatting the results for use in a prompt depends a bit on the use case. By +default, the results will be sorted by score, not necessarily in the order they +appear in the original text. You may want to sort them by the order they appear +in the original text so they follow the flow of the original document. For convenience, the `text` field of the search results is a string formatted with `...` separating non-sequential chunks, `---` separating entries, and @@ -300,8 +305,8 @@ Chunk 5 contents ``` There is also a `text` field on each entry that is the full text of the entry, -similarly formatted with `...` separating non-sequential chunks, if you want -to format each entry differently. +similarly formatted with `...` separating non-sequential chunks, if you want to +format each entry differently. For a fully custom format, you can use the `results` field and entries directly: @@ -350,33 +355,33 @@ await generateText({ ## Using keys to gracefully replace content -When you add content to a namespace, you can provide a `key` to uniquely identify the content. -If you add content with the same key, it will make a new entry to replace the old one. +When you add content to a namespace, you can provide a `key` to uniquely +identify the content. If you add content with the same key, it will make a new +entry to replace the old one. ```ts await rag.add(ctx, { namespace: userId, key: "my-file.txt", text }); ``` -When a new document is added, it will start with a status of "pending" while -it chunks, embeds, and inserts the data into the database. -Once all data is inserted, it will iterate over the chunks and swap the old -content embeddings with the new ones, and then update the status to "ready", -marking the previous version as "replaced". +When a new document is added, it will start with a status of "pending" while it +chunks, embeds, and inserts the data into the database. Once all data is +inserted, it will iterate over the chunks and swap the old content embeddings +with the new ones, and then update the status to "ready", marking the previous +version as "replaced". The old content is kept around by default, so in-flight searches will get -results for old vector search results. -See below for more details on deleting. +results for old vector search results. See below for more details on deleting. This means that if searches are happening while the document is being added, -they will see the old content results -This is useful if you want to add content to a namespace and then immediately -search for it, or if you want to add content to a namespace and then immediately -add more content to the same namespace. +they will see the old content results This is useful if you want to add content +to a namespace and then immediately search for it, or if you want to add content +to a namespace and then immediately add more content to the same namespace. ## Using your own content splitter -By default, the component uses the `defaultChunker` to split the content into chunks. -You can pass in your own content chunks to the `add` or `addAsync` functions. +By default, the component uses the `defaultChunker` to split the content into +chunks. You can pass in your own content chunks to the `add` or `addAsync` +functions. ```ts const chunks = await textSplitter.split(content); @@ -386,8 +391,8 @@ await rag.add(ctx, { namespace: "global", chunks }); Note: The `textSplitter` here could be LangChain, Mastra, or something custom. The simplest version makes an array of strings like `content.split("\n")`. -Note: you can pass in an async iterator instead of an array to handle large content. -Or use the `addAsync` function (see below). +Note: you can pass in an async iterator instead of an array to handle large +content. Or use the `addAsync` function (see below). ## Providing custom embeddings per-chunk @@ -404,7 +409,7 @@ const chunksWithEmbeddings = await Promise.all( ...chunk, embedding: await embedSummary(chunk), }; - }) + }), ); await rag.add(ctx, { namespace: "global", chunks }); ``` @@ -451,14 +456,14 @@ export const chunkerAction = rag.defineChunkerAction(async (ctx, args) => { export default cors.http; ``` -You can upload files directly to a Convex action, httpAction, or upload url. -See the [docs](https://docs.convex.dev/file-storage/upload-files) for details. +You can upload files directly to a Convex action, httpAction, or upload url. See +the [docs](https://docs.convex.dev/file-storage/upload-files) for details. ### OnComplete Handling You can register an `onComplete` handler when adding content that will be called -when the entry was created and is ready, or if there was an error or it was replaced before it -finished. +when the entry was created and is ready, or if there was an error or it was +replaced before it finished. ```ts // in an action @@ -476,12 +481,14 @@ export const docComplete = rag.defineOnComplete( } // You can associate the entry with your own data here. This will commit // in the same transaction as the entry becoming ready. - } + }, ); ``` -Note: The `onComplete` callback is only triggered when new content is processed. If you add content that already exists (`contentHash` did not change for the same `key`), `onComplete` will not be called. -To handle this case, you can check the return value of `rag.add()`: +Note: The `onComplete` callback is only triggered when new content is processed. +If you add content that already exists (`contentHash` did not change for the +same `key`), `onComplete` will not be called. To handle this case, you can check +the return value of `rag.add()`: ```ts const { status, created } = await rag.add(ctx, { @@ -502,8 +509,8 @@ if (status === "ready" && !created) { Here's a simple example fetching content from a URL to add. -It also adds filters to the entry, so you can search for it later by -category, contentType, or both. +It also adds filters to the entry, so you can search for it later by category, +contentType, or both. ```ts export const add = action({ @@ -581,7 +588,7 @@ crons.interval( "deleteOldContent", { hours: 1 }, internal.crons.deleteOldContent, - {} + {}, ); export default crons; ``` @@ -604,13 +611,13 @@ Types for the various elements: - While the `EntryId` and `NamespaceId` are strings under the hood, they are given more specific types to make it easier to use them correctly. -Validators can be used in `args` and schema table definitions: -`vEntry`, `vEntryId`, `vNamespaceId`, `vSearchEntry`, `vSearchResult` +Validators can be used in `args` and schema table definitions: `vEntry`, +`vEntryId`, `vNamespaceId`, `vSearchEntry`, `vSearchResult` e.g. `defineTable({ myDocTitle: v.string(), entryId: vEntryId })` -The validators for the branded IDs will only validate they are strings, -but will have the more specific types, to provide type safety. +The validators for the branded IDs will only validate they are strings, but will +have the more specific types, to provide type safety. ## Utility Functions @@ -665,29 +672,29 @@ const results = hybridRank( [textSearchResults, vectorSearchResults, recentSearchResults], { weights: [2, 1, 3], // prefer recent results more than text or vector - } + }, ); // results = [ id3, id5, id1, id2, id4 ] ``` -To have it more biased towards the top few results, you can set the `k` value -to a lower number (10 by default). +To have it more biased towards the top few results, you can set the `k` value to +a lower number (10 by default). ```ts const results = hybridRank( [textSearchResults, vectorSearchResults, recentSearchResults], - { k: 1 } + { k: 1 }, ); // results = [ id5, id1, id3, id2, id4 ] ``` ### `contentHashFromArrayBuffer` -This generates the hash of a file's contents, which can be used to avoid -adding the same file twice. +This generates the hash of a file's contents, which can be used to avoid adding +the same file twice. -Note: doing `blob.arrayBuffer()` will consume the blob's data, so you'll need -to make a new blob to use it after calling this function. +Note: doing `blob.arrayBuffer()` will consume the blob's data, so you'll need to +make a new blob to use it after calling this function. ```ts import { contentHashFromArrayBuffer } from "@convex-dev/rag"; diff --git a/convex.json b/convex.json index 07199a6..5dbf6e9 100644 --- a/convex.json +++ b/convex.json @@ -1,7 +1,7 @@ { - "$schema": "https://raw.githubusercontent.com/get-convex/convex-backend/refs/heads/main/npm-packages/convex/schemas/convex.schema.json", + "$schema": "./node_modules/convex/schemas/convex.schema.json", "functions": "example/convex", "codegen": { - "staticApi": true + "legacyComponentApi": true } } diff --git a/eslint.config.js b/eslint.config.js index 0c7323c..075dc41 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,59 +1,47 @@ import globals from "globals"; import pluginJs from "@eslint/js"; import tseslint from "typescript-eslint"; -import reactPlugin from "eslint-plugin-react"; import reactHooks from "eslint-plugin-react-hooks"; +import reactRefresh from "eslint-plugin-react-refresh"; export default [ - { files: ["src/**/*.{js,mjs,cjs,ts,tsx}"] }, { ignores: [ "dist/**", - "eslint.config.js", + "example/dist/**", + "example/public/**", + "*.config.{js,mjs,cjs,ts,tsx}", + "example/**/*.config.{js,mjs,cjs,ts,tsx}", + "vitest.config.ts", "**/_generated/", - "node10stubs.mjs", ], }, { + files: ["src/**/*.{js,mjs,cjs,ts,tsx}", "example/**/*.{js,mjs,cjs,ts,tsx}"], languageOptions: { - globals: globals.worker, parser: tseslint.parser, - parserOptions: { - project: true, + project: [ + "./tsconfig.json", + "./example/tsconfig.json", + "./example/convex/tsconfig.json", + ], tsconfigRootDir: import.meta.dirname, }, }, }, pluginJs.configs.recommended, ...tseslint.configs.recommended, + // Convex code - Worker environment { - files: [ - "src/react/**/*.{jsx,tsx}", - "src/react/**/*.js", - "src/react/**/*.ts", - ], - plugins: { react: reactPlugin, "react-hooks": reactHooks }, - settings: { - react: { - version: "detect", - }, + files: ["src/**/*.{ts,tsx}", "example/convex/**/*.{ts,tsx}"], + ignores: ["src/react/**"], + languageOptions: { + globals: globals.worker, }, - rules: { - ...reactPlugin.configs["recommended"].rules, - "react/jsx-uses-react": "off", - "react/react-in-jsx-scope": "off", - "react/prop-types": "off", - "react-hooks/rules-of-hooks": "error", - "react-hooks/exhaustive-deps": "warn", - }, - }, - { rules: { "@typescript-eslint/no-floating-promises": "error", - "eslint-comments/no-unused-disable": "off", - - // allow (_arg: number) => {} and const _foo = 1; + "@typescript-eslint/no-explicit-any": "off", "no-unused-vars": "off", "@typescript-eslint/no-unused-vars": [ "warn", @@ -62,8 +50,6 @@ export default [ varsIgnorePattern: "^_", }, ], - - // Fix the no-unused-expressions rule configuration "@typescript-eslint/no-unused-expressions": [ "error", { @@ -74,4 +60,42 @@ export default [ ], }, }, + // React app code - Browser environment + { + files: ["src/react/**/*.{ts,tsx}", "example/src/**/*.{ts,tsx}"], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + plugins: { + "react-hooks": reactHooks, + "react-refresh": reactRefresh, + }, + rules: { + ...reactHooks.configs.recommended.rules, + "react-refresh/only-export-components": [ + "warn", + { allowConstantExport: true }, + ], + "@typescript-eslint/no-explicit-any": "off", + "no-unused-vars": "off", + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + }, + ], + }, + }, + // Example config files (vite.config.ts, etc.) - Node environment + { + files: ["example/vite.config.ts", "example/**/*.config.{js,ts}"], + languageOptions: { + globals: { + ...globals.node, + ...globals.browser, + }, + }, + }, ]; diff --git a/example/.gitignore b/example/.gitignore deleted file mode 100644 index 983c26f..0000000 --- a/example/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -!**/glob-import/dir/node_modules -.DS_Store -.idea -*.cpuprofile -*.local -*.log -/.vscode/ -/docs/.vitepress/cache -dist -dist-ssr -explorations -node_modules -playground-temp -temp -TODOs.md -.eslintcache diff --git a/example/convex/README.md b/example/convex/README.md new file mode 100644 index 0000000..7fda0c3 --- /dev/null +++ b/example/convex/README.md @@ -0,0 +1,90 @@ +# Welcome to your Convex functions directory! + +Write your Convex functions here. +See https://docs.convex.dev/functions for more. + +A query function that takes two arguments looks like: + +```ts +// convex/myFunctions.ts +import { query } from "./_generated/server"; +import { v } from "convex/values"; + +export const myQueryFunction = query({ + // Validators for arguments. + args: { + first: v.number(), + second: v.string(), + }, + + // Function implementation. + handler: async (ctx, args) => { + // Read the database as many times as you need here. + // See https://docs.convex.dev/database/reading-data. + const documents = await ctx.db.query("tablename").collect(); + + // Arguments passed from the client are properties of the args object. + console.log(args.first, args.second); + + // Write arbitrary JavaScript here: filter, aggregate, build derived data, + // remove non-public properties, or create new objects. + return documents; + }, +}); +``` + +Using this query function in a React component looks like: + +```ts +const data = useQuery(api.myFunctions.myQueryFunction, { + first: 10, + second: "hello", +}); +``` + +A mutation function looks like: + +```ts +// convex/myFunctions.ts +import { mutation } from "./_generated/server"; +import { v } from "convex/values"; + +export const myMutationFunction = mutation({ + // Validators for arguments. + args: { + first: v.string(), + second: v.string(), + }, + + // Function implementation. + handler: async (ctx, args) => { + // Insert or modify documents in the database here. + // Mutations can also read from the database like queries. + // See https://docs.convex.dev/database/writing-data. + const message = { body: args.first, author: args.second }; + const id = await ctx.db.insert("messages", message); + + // Optionally, return a value from your mutation. + return await ctx.db.get(id); + }, +}); +``` + +Using this mutation function in a React component looks like: + +```ts +const mutation = useMutation(api.myFunctions.myMutationFunction); +function handleButtonPress() { + // fire and forget, the most common way to use mutations + mutation({ first: "Hello!", second: "me" }); + // OR + // use the result once the mutation has completed + mutation({ first: "Hello!", second: "me" }).then((result) => + console.log(result), + ); +} +``` + +Use the Convex CLI to push your functions to a deployment. See everything +the Convex CLI can do by running `npx convex -h` in your project root +directory. To learn more, launch the docs with `npx convex docs`. diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index 616f56f..1b464f4 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -8,8 +8,23 @@ * @module */ -import type { FunctionReference } from "convex/server"; -import type { GenericId as Id } from "convex/values"; +import type * as crons from "../crons.js"; +import type * as example from "../example.js"; +import type * as getText from "../getText.js"; +import type * as http from "../http.js"; + +import type { + ApiFromModules, + FilterApi, + FunctionReference, +} from "convex/server"; + +declare const fullApi: ApiFromModules<{ + crons: typeof crons; + example: typeof example; + getText: typeof getText; + http: typeof http; +}>; /** * A utility for referencing Convex functions in your app's public API. @@ -19,112 +34,10 @@ import type { GenericId as Id } from "convex/values"; * const myFunctionReference = api.myModule.myFunction; * ``` */ -export declare const api: { - example: { - addFile: FunctionReference< - "action", - "public", - { - bytes: ArrayBuffer; - category?: string; - filename: string; - globalNamespace: boolean; - mimeType: string; - }, - any - >; - askQuestion: FunctionReference< - "action", - "public", - { - chunkContext?: { after: number; before: number }; - filter?: - | { name: "category"; value: null | string } - | { name: "filename"; value: string }; - globalNamespace: boolean; - limit?: number; - prompt: string; - }, - any - >; - deleteFile: FunctionReference< - "mutation", - "public", - { entryId: string }, - any - >; - listChunks: FunctionReference< - "query", - "public", - { - entryId: string; - order: "desc" | "asc"; - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - }, - any - >; - listFiles: FunctionReference< - "query", - "public", - { - category?: string; - globalNamespace: boolean; - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - }, - any - >; - listPendingFiles: FunctionReference<"query", "public", {}, any>; - search: FunctionReference< - "action", - "public", - { - chunkContext?: { after: number; before: number }; - globalNamespace: boolean; - limit?: number; - query: string; - }, - any - >; - searchCategory: FunctionReference< - "action", - "public", - { - category: string; - chunkContext?: { after: number; before: number }; - globalNamespace: boolean; - limit?: number; - query: string; - }, - any - >; - searchFile: FunctionReference< - "action", - "public", - { - chunkContext?: { after: number; before: number }; - filename: string; - globalNamespace: boolean; - limit?: number; - query: string; - }, - any - >; - }; -}; +export declare const api: FilterApi< + typeof fullApi, + FunctionReference +>; /** * A utility for referencing Convex functions in your app's internal API. @@ -134,85 +47,10 @@ export declare const api: { * const myFunctionReference = internal.myModule.myFunction; * ``` */ -export declare const internal: { - example: { - chunkerAction: FunctionReference< - "action", - "internal", - { - entry: { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }; - insertChunks: string; - namespace: { - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }; - }, - any - >; - deleteOldContent: FunctionReference< - "mutation", - "internal", - { cursor?: string }, - any - >; - recordUploadMetadata: FunctionReference< - "mutation", - "internal", - { - entry: { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }; - error?: string; - namespace: { - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }; - replacedEntry?: { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }; - }, - any - >; - }; -}; +export declare const internal: FilterApi< + typeof fullApi, + FunctionReference +>; export declare const components: { rag: { diff --git a/example/convex/_generated/server.d.ts b/example/convex/_generated/server.d.ts index b5c6828..bec05e6 100644 --- a/example/convex/_generated/server.d.ts +++ b/example/convex/_generated/server.d.ts @@ -10,7 +10,6 @@ import { ActionBuilder, - AnyComponents, HttpActionBuilder, MutationBuilder, QueryBuilder, @@ -19,15 +18,9 @@ import { GenericQueryCtx, GenericDatabaseReader, GenericDatabaseWriter, - FunctionReference, } from "convex/server"; import type { DataModel } from "./dataModel.js"; -type GenericCtx = - | GenericActionCtx - | GenericMutationCtx - | GenericQueryCtx; - /** * Define a query in this Convex app's public API. * @@ -92,11 +85,12 @@ export declare const internalAction: ActionBuilder; /** * Define an HTTP action. * - * This function will be used to respond to HTTP requests received by a Convex - * deployment if the requests matches the path and method where this action - * is routed. Be sure to route your action in `convex/http.js`. + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. * - * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. */ export declare const httpAction: HttpActionBuilder; diff --git a/example/convex/_generated/server.js b/example/convex/_generated/server.js index 4a21df4..bf3d25a 100644 --- a/example/convex/_generated/server.js +++ b/example/convex/_generated/server.js @@ -16,7 +16,6 @@ import { internalActionGeneric, internalMutationGeneric, internalQueryGeneric, - componentsGeneric, } from "convex/server"; /** @@ -81,10 +80,14 @@ export const action = actionGeneric; export const internalAction = internalActionGeneric; /** - * Define a Convex HTTP action. + * Define an HTTP action. * - * @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object - * as its second. - * @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. */ export const httpAction = httpActionGeneric; diff --git a/example/convex/crons.ts b/example/convex/crons.ts index 87864e1..02f7255 100644 --- a/example/convex/crons.ts +++ b/example/convex/crons.ts @@ -7,7 +7,7 @@ crons.interval( "deleteOldContent", { hours: 1 }, internal.example.deleteOldContent, - {} + {}, ); export default crons; diff --git a/example/convex/example.ts b/example/convex/example.ts index 349dd01..f86cf7e 100644 --- a/example/convex/example.ts +++ b/example/convex/example.ts @@ -89,7 +89,7 @@ export const search = action({ globalNamespace: v.boolean(), limit: v.optional(v.number()), chunkContext: v.optional( - v.object({ before: v.number(), after: v.number() }) + v.object({ before: v.number(), after: v.number() }), ), }, handler: async (ctx, args) => { @@ -112,7 +112,7 @@ export const searchFile = action({ filename: v.string(), limit: v.optional(v.number()), chunkContext: v.optional( - v.object({ before: v.number(), after: v.number() }) + v.object({ before: v.number(), after: v.number() }), ), }, handler: async (ctx, args) => { @@ -138,7 +138,7 @@ export const searchCategory = action({ category: v.string(), limit: v.optional(v.number()), chunkContext: v.optional( - v.object({ before: v.number(), after: v.number() }) + v.object({ before: v.number(), after: v.number() }), ), }, handler: async (ctx, args) => { @@ -167,12 +167,12 @@ export const askQuestion = action({ name: v.literal("category"), value: v.union(v.null(), v.string()), }), - v.object({ name: v.literal("filename"), value: v.string() }) - ) + v.object({ name: v.literal("filename"), value: v.string() }), + ), ), limit: v.optional(v.number()), chunkContext: v.optional( - v.object({ before: v.number(), after: v.number() }) + v.object({ before: v.number(), after: v.number() }), ), }, handler: async (ctx, args) => { @@ -208,7 +208,7 @@ export async function addFileAsync( filename: string; blob: Blob; category: string | null; - } + }, ) { const userId = await getUserId(ctx); // Maybe rate limit how often a user can upload a file / attribute? @@ -228,7 +228,7 @@ export async function addFileAsync( } // If it doesn't exist, we need to store the file and chunk it asynchronously. const storageId = await ctx.storage.store( - new Blob([bytes], { type: blob.type }) + new Blob([bytes], { type: blob.type }), ); const { entryId } = await rag.addAsync(ctx, { namespace, @@ -284,7 +284,7 @@ export const listFiles = query({ return { ...results, page: await Promise.all( - results.page.map((entry) => toFile(ctx, entry, args.globalNamespace)) + results.page.map((entry) => toFile(ctx, entry, args.globalNamespace)), ), }; }, @@ -338,7 +338,7 @@ export type PublicFile = { async function toFiles( ctx: ActionCtx, - files: SearchEntry[] + files: SearchEntry[], ): Promise { return await Promise.all(files.map((entry) => toFile(ctx, entry, false))); } @@ -346,7 +346,7 @@ async function toFiles( async function toFile( ctx: { storage: StorageReader }, entry: Entry, - global: boolean + global: boolean, ): Promise { assert(entry.metadata, "Entry metadata not found"); const storageId = entry.metadata.storageId; @@ -419,7 +419,7 @@ export const recordUploadMetadata = rag.defineOnComplete( console.debug("adding file failed", entry, error); await rag.deleteAsync(ctx, { entryId: entry.entryId }); } - } + }, ); export const deleteFile = mutation({ @@ -439,7 +439,7 @@ async function _deleteFile(ctx: MutationCtx, entryId: EntryId) { if (file) { await ctx.db.delete(file._id); await ctx.storage.delete(file.storageId); - await rag.delete(ctx, { entryId }); + await rag.deleteAsync(ctx, { entryId }); } } diff --git a/example/convex/getText.ts b/example/convex/getText.ts index 429bb84..9cd6ea3 100644 --- a/example/convex/getText.ts +++ b/example/convex/getText.ts @@ -21,7 +21,7 @@ export async function getText( filename: string; bytes?: ArrayBuffer; mimeType: string; - } + }, ) { const url = await ctx.storage.getUrl(storageId); assert(url); diff --git a/example/convex/tsconfig.json b/example/convex/tsconfig.json index 515aa99..891d309 100644 --- a/example/convex/tsconfig.json +++ b/example/convex/tsconfig.json @@ -1,33 +1,5 @@ { - /* This TypeScript project config describes the environment that - * Convex functions run in and is used to typecheck them. - * You can modify it, but some settings required to use Convex. - */ - "compilerOptions": { - /* These settings are not required by Convex and can be modified. */ - "allowJs": true, - "strict": true, - "skipLibCheck": true, - - /* These compiler options are required by Convex */ - "target": "ESNext", - "lib": ["ES2021", "dom", "ESNext.Array"], - "forceConsistentCasingInFileNames": true, - "allowSyntheticDefaultImports": true, - "module": "ESNext", - "moduleResolution": "Bundler", - "isolatedModules": true, - "noEmit": true - - /* This should only be used in this example. Real apps should not attempt - * to compile TypeScript because differences between tsconfig.json files can - * cause the code to be compiled differently. - */ - // Un-comment this to get instant types between your component and example. - // However, if you're willing to wait for a build before the types update, - // it's better to leave this commented out to catch build errors faster. - // "customConditions": ["@convex-dev/component-source"] - }, - "include": ["./**/*"], - "exclude": ["./_generated"] + "extends": "../tsconfig.json", + "include": ["."], + "exclude": ["_generated"] } diff --git a/example/eslint.config.js b/example/eslint.config.js deleted file mode 100644 index f0c101e..0000000 --- a/example/eslint.config.js +++ /dev/null @@ -1,40 +0,0 @@ -import js from "@eslint/js"; -import globals from "globals"; -import reactHooks from "eslint-plugin-react-hooks"; -import reactRefresh from "eslint-plugin-react-refresh"; -import tseslint from "typescript-eslint"; - -export default tseslint.config( - { ignores: ["dist"] }, - { - extends: [js.configs.recommended, ...tseslint.configs.recommended], - files: ["**/*.{ts,tsx}"], - ignores: ["convex"], - languageOptions: { - ecmaVersion: 2020, - globals: globals.browser, - }, - plugins: { - "react-hooks": reactHooks, - "react-refresh": reactRefresh, - }, - rules: { - ...reactHooks.configs.recommended.rules, - // Allow explicit `any`s - "@typescript-eslint/no-explicit-any": "off", - "react-refresh/only-export-components": [ - "warn", - { allowConstantExport: true }, - ], - - "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": [ - "warn", - { - argsIgnorePattern: "^_", - varsIgnorePattern: "^_", - }, - ], - }, - } -); diff --git a/example/src/Example.tsx b/example/src/Example.tsx index 3c0e668..04da7ab 100644 --- a/example/src/Example.tsx +++ b/example/src/Example.tsx @@ -1,5 +1,5 @@ import "./Example.css"; -import { useQuery, useConvex } from "convex/react"; +import { useConvex } from "convex/react"; import { usePaginatedQuery } from "convex-helpers/react"; import { api } from "../convex/_generated/api"; import { useCallback, useState, useEffect } from "react"; @@ -34,17 +34,16 @@ function Example() { const [searchGlobal, setSearchGlobal] = useState(true); const [searchQuery, setSearchQuery] = useState(""); const [selectedDocument, setSelectedDocument] = useState( - null + null, ); const [selectedCategory, setSelectedCategory] = useState(""); const [searchResults, setSearchResults] = useState( - null + null, ); const [questionResult, setQuestionResult] = useState( - null + null, ); const [isSearching, setIsSearching] = useState(false); - const [showChunks, setShowChunks] = useState(false); const [categorySearchGlobal, setCategorySearchGlobal] = useState(true); const [showFullText, setShowFullText] = useState(false); @@ -65,7 +64,7 @@ function Example() { order: "asc", } : "skip", - { initialNumItems: 10 } + { initialNumItems: 10 }, ); const handleSearch = useCallback( @@ -186,7 +185,7 @@ function Example() { } catch (error) { console.error("Search/Question failed:", error); alert( - `${mode === "question" ? "Question" : "Search"} failed. ${error instanceof Error ? error.message : String(error)}` + `${mode === "question" ? "Question" : "Search"} failed. ${error instanceof Error ? error.message : String(error)}`, ); } finally { setIsSearching(false); @@ -203,7 +202,7 @@ function Example() { limit, chunksBefore, chunksAfter, - ] + ], ); const handleFileSelect = (file: PublicFile | null) => { @@ -271,7 +270,7 @@ function Example() { {/* Results Area */}
{/* Question Results */} - {questionResult && (searchType !== "file" || !showChunks) && ( + {questionResult && (
{/* Generated Answer */}
@@ -296,7 +295,7 @@ function Example() { {searchType === "file" && selectedDocument && documentChunks.status !== "LoadingFirstPage" && - (showChunks || !searchResults) && ( + !searchResults && (
@@ -404,7 +403,7 @@ function Example() { )} {/* Search Results */} - {searchResults && (searchType !== "file" || !showChunks) && ( + {searchResults && (
{/* Sources Section */}
@@ -623,53 +622,46 @@ function Example() { )} {/* Empty State */} - {!searchResults && - !questionResult && - !( - searchType === "file" && - selectedDocument && - documentChunks && - showChunks - ) && ( -
- {isSearching ? ( -
-
-

- Searching... -

-

- Please wait while we search your documents -

+ {!searchResults && !questionResult && ( +
+ {isSearching ? ( +
+
+

+ Searching... +

+

+ Please wait while we search your documents +

+
+ ) : ( + <> +
+ + +
- ) : ( - <> -
- - - -
-

- Ready to Search or Ask -

-

- Use the 🔍 button to search your documents or the Ask - button to get AI-generated answers using search context. -

- - )} -
- )} +

+ Ready to Search or Ask +

+

+ Use the 🔍 button to search your documents or the Ask button + to get AI-generated answers using search context. +

+ + )} +
+ )}
diff --git a/example/src/components/FileList.tsx b/example/src/components/FileList.tsx index 7e257b9..fd8f456 100644 --- a/example/src/components/FileList.tsx +++ b/example/src/components/FileList.tsx @@ -28,7 +28,7 @@ function PendingDocumentProgress({ doc }: { doc: PublicFile }) { // Find first chunk with state "ready" to get live count const firstReadyChunk = chunks.page.find( - (chunk) => chunk.state === "ready" + (chunk) => chunk.state === "ready", ); const live = firstReadyChunk ? firstReadyChunk.order + 1 : 0; @@ -135,7 +135,7 @@ export function FileList({ { globalNamespace: true, }, - { initialNumItems: 10 } + { initialNumItems: 10 }, ); const userFiles = usePaginatedQuery( @@ -143,7 +143,7 @@ export function FileList({ { globalNamespace: false, }, - { initialNumItems: 10 } + { initialNumItems: 10 }, ); const pendingFiles = useQuery(api.example.listPendingFiles); @@ -162,20 +162,20 @@ export function FileList({ } catch (error) { console.error("Delete failed:", error); alert( - `Failed to delete entry. ${error instanceof Error ? error.message : String(error)}` + `Failed to delete entry. ${error instanceof Error ? error.message : String(error)}`, ); } }, - [convex, selectedDocument, onFileSelect] + [convex, selectedDocument, onFileSelect], ); useEffect(() => { const categories = new Set(); globalFiles?.results?.forEach( - (doc) => doc.category && categories.add(doc.category) + (doc) => doc.category && categories.add(doc.category), ); userFiles?.results?.forEach( - (doc) => doc.category && categories.add(doc.category) + (doc) => doc.category && categories.add(doc.category), ); onCategoriesChange(Array.from(categories).sort()); }, [globalFiles?.results, userFiles?.results, onCategoriesChange]); diff --git a/example/src/components/UploadSection.tsx b/example/src/components/UploadSection.tsx index f7fdd85..0b8ce5b 100644 --- a/example/src/components/UploadSection.tsx +++ b/example/src/components/UploadSection.tsx @@ -75,7 +75,7 @@ export function UploadSection({ onFileUploaded }: UploadSectionProps) { } } }, - [uploadForm.filename] + [uploadForm.filename], ); const handleFileClear = useCallback(() => { @@ -88,7 +88,7 @@ export function UploadSection({ onFileUploaded }: UploadSectionProps) { }); // Clear file input const fileInput = document.querySelector( - 'input[type="file"]' + 'input[type="file"]', ) as HTMLInputElement; if (fileInput) fileInput.value = ""; }, []); @@ -102,7 +102,7 @@ export function UploadSection({ onFileUploaded }: UploadSectionProps) { // For PDFs with extraction errors, ask user if they want to proceed if (selectedFile && isPdfFile(selectedFile) && pdfExtraction.error) { const proceed = confirm( - `PDF text extraction failed: ${pdfExtraction.error}\n\nDo you want to upload the PDF file directly instead?` + `PDF text extraction failed: ${pdfExtraction.error}\n\nDo you want to upload the PDF file directly instead?`, ); if (!proceed) return; } @@ -161,7 +161,7 @@ export function UploadSection({ onFileUploaded }: UploadSectionProps) { // Clear file input const fileInput = document.querySelector( - 'input[type="file"]' + 'input[type="file"]', ) as HTMLInputElement; if (fileInput) fileInput.value = ""; @@ -174,7 +174,7 @@ export function UploadSection({ onFileUploaded }: UploadSectionProps) { })); setSelectedFile(selectedFile); alert( - `Upload failed. ${error instanceof Error ? error.message : String(error)}` + `Upload failed. ${error instanceof Error ? error.message : String(error)}`, ); } finally { setIsAdding(false); diff --git a/example/src/main.tsx b/example/src/main.tsx index b757465..80cacd9 100644 --- a/example/src/main.tsx +++ b/example/src/main.tsx @@ -1,7 +1,7 @@ import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { ConvexProvider, ConvexReactClient } from "convex/react"; -import Example from "./Example.tsx"; +import Example from "./Example.jsx"; import "./index.css"; const address = import.meta.env.VITE_CONVEX_URL; @@ -13,5 +13,5 @@ createRoot(document.getElementById("root")!).render( - + , ); diff --git a/example/src/pdfUtils.ts b/example/src/pdfUtils.ts index 60baff2..8164432 100644 --- a/example/src/pdfUtils.ts +++ b/example/src/pdfUtils.ts @@ -12,7 +12,7 @@ export interface PdfExtractionResult { } export async function extractTextFromPdf( - file: File + file: File, ): Promise { try { const arrayBuffer = await file.arrayBuffer(); @@ -57,7 +57,7 @@ export async function extractTextFromPdf( } catch (error) { console.error("Error extracting text from PDF:", error); throw new Error( - "Failed to extract text from PDF. The file may be corrupted or password-protected." + "Failed to extract text from PDF. The file may be corrupted or password-protected.", ); } } diff --git a/example/tsconfig.json b/example/tsconfig.json index a95a348..094a019 100644 --- a/example/tsconfig.json +++ b/example/tsconfig.json @@ -10,18 +10,8 @@ "moduleResolution": "Bundler", "resolveJsonModule": true, "isolatedModules": true, - "allowImportingTsExtensions": true, - "noEmit": true, - "jsx": "react-jsx" - - /* This should only be used in this example. Real apps should not attempt - * to compile TypeScript because differences between tsconfig.json files can - * cause the code to be compiled differently. - */ - // Un-comment this to get instant types between your component and example. - // However, if you're willing to wait for a build before the types update, - // it's better to leave this commented out to catch build errors faster. - // "customConditions": ["@convex-dev/component-source"] + "jsx": "react-jsx", + "noEmit": true }, "include": ["./src", "vite.config.ts"] } diff --git a/example/vite.config.ts b/example/vite.config.ts index 0ad2283..8ecf715 100644 --- a/example/vite.config.ts +++ b/example/vite.config.ts @@ -3,11 +3,9 @@ import react from "@vitejs/plugin-react"; // https://vitejs.dev/config/ export default defineConfig({ + envDir: "../", plugins: [react()], optimizeDeps: { include: ["pdfjs-dist"], }, - resolve: { - conditions: ["@convex-dev/component-source"], - }, }); diff --git a/package-lock.json b/package-lock.json index 083e83c..140525d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@convex-dev/rag", - "version": "0.5.4", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@convex-dev/rag", - "version": "0.5.4", + "version": "0.6.0", "license": "Apache-2.0", "dependencies": { "ai": "^5.0.0" @@ -14,7 +14,7 @@ "devDependencies": { "@ai-sdk/openai": "2.0.59", "@arethetypeswrong/cli": "0.18.2", - "@convex-dev/workpool": "0.2.19", + "@convex-dev/workpool": "0.3.0", "@edge-runtime/vm": "5.0.0", "@eslint/eslintrc": "3.3.1", "@eslint/js": "9.39.0", @@ -22,7 +22,7 @@ "@tailwindcss/postcss": "4.1.16", "@tailwindcss/typography": "0.5.19", "@types/eslint-plugin-react-refresh": "0.4.0", - "@types/node": "18.19.130", + "@types/node": "20.19.24", "@types/react": "19.2.2", "@types/react-dom": "19.2.2", "@typescript-eslint/eslint-plugin": "8.46.2", @@ -31,6 +31,7 @@ "ai": "5.0.86", "autoprefixer": "10.4.21", "chokidar-cli": "3.0.0", + "convex": "1.29.0", "convex-helpers": "0.1.104", "convex-test": "0.0.38", "cpy-cli": "6.0.0", @@ -55,7 +56,7 @@ }, "peerDependencies": { "@convex-dev/workpool": "^0.2.14", - "convex": ">=1.25.0 <1.35.0", + "convex": "^1.24.8", "convex-helpers": "^0.1.94" } }, @@ -573,10 +574,11 @@ } }, "node_modules/@braidai/lang": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@braidai/lang/-/lang-1.1.1.tgz", - "integrity": "sha512-5uM+no3i3DafVgkoW7ayPhEGHNNBZCSj5TrGDQt0ayEKQda5f3lAXlmQg0MR5E0gKgmTzUUEtSWHsEC3h9jUcg==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@braidai/lang/-/lang-1.1.2.tgz", + "integrity": "sha512-qBcknbBufNHlui137Hft8xauQMTZDKdophmLFv05r2eNmdIv/MlPuP4TdUknHG68UdWLgVZwgxVe735HzJNIwA==", + "dev": true, + "license": "ISC" }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", @@ -597,13 +599,13 @@ } }, "node_modules/@convex-dev/workpool": { - "version": "0.2.19", - "resolved": "https://registry.npmjs.org/@convex-dev/workpool/-/workpool-0.2.19.tgz", - "integrity": "sha512-U2KwYnsKILyxW1baWEhDv+ZtnL5FZbYFxBT5owQ0Lw/kseiudMZraA4clH+/6gowHSahWpkq4wndhcOfpfhuOA==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@convex-dev/workpool/-/workpool-0.3.0.tgz", + "integrity": "sha512-nY8Ub0pmfuxZ2rcnNwVeESYPyJqLU4h+afodEdg8Ifnr+vcFUuee/p69vMFmeqC2y4yo9IDPHrdiVZVyjbBE7A==", "dev": true, "license": "Apache-2.0", "peerDependencies": { - "convex": ">=1.25.0 <1.35.0", + "convex": "^1.24.8", "convex-helpers": "^0.1.94" } }, @@ -638,6 +640,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -654,6 +657,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -670,6 +674,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -686,6 +691,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -702,6 +708,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -718,6 +725,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -734,6 +742,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -750,6 +759,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -766,6 +776,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -782,6 +793,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -798,6 +810,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -814,6 +827,7 @@ "cpu": [ "loong64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -830,6 +844,7 @@ "cpu": [ "mips64el" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -846,6 +861,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -862,6 +878,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -878,6 +895,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -894,6 +912,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -910,6 +929,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -926,6 +946,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -942,6 +963,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -958,6 +980,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -974,6 +997,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -990,6 +1014,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1006,6 +1031,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1022,6 +1048,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1376,9 +1403,9 @@ } }, "node_modules/@langchain/core": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.0.2.tgz", - "integrity": "sha512-6mOn4bZyO6XT0GGrEijRtMVrmYJGZ8y1BcwyTPDptFz38lP0CEzrKEYB++h+u3TEcAd3eO25l1aGw/zVlVgw2Q==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.0.4.tgz", + "integrity": "sha512-zrTM4sVls18KfR6h/R7ErJUY4eeZa3Mr9s+Y6upXc2MevlYo7jfZZabs4Kv/R9fTdRFEJPwSsY1HTw5pokPrLg==", "dev": true, "license": "MIT", "dependencies": { @@ -2511,66 +2538,6 @@ "node": ">=14.0.0" } }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.5.0", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.1.0", - "tslib": "^2.4.0" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.5.0", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { - "version": "1.0.7", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.5.0", - "@emnapi/runtime": "^1.5.0", - "@tybys/wasm-util": "^0.10.1" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { - "version": "2.8.1", - "dev": true, - "inBundle": true, - "license": "0BSD", - "optional": true - }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.1.16", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.16.tgz", @@ -2777,14 +2744,14 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "version": "20.19.24", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.24.tgz", + "integrity": "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/react": { @@ -3279,9 +3246,9 @@ } }, "node_modules/ansi-escapes": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.0.0.tgz", - "integrity": "sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.2.0.tgz", + "integrity": "sha512-g6LhBsl+GBPRWGWsBtutpzBYuIIdBkLEvad5C/va/74Db018+5TZiyA26cZJAr3Rft5lprVqOIPxf5Vid6tqAw==", "dev": true, "license": "MIT", "dependencies": { @@ -3295,9 +3262,9 @@ } }, "node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "dev": true, "license": "MIT", "engines": { @@ -4188,13 +4155,13 @@ "license": "MIT" }, "node_modules/console-table-printer": { - "version": "2.14.6", - "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.14.6.tgz", - "integrity": "sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.15.0.tgz", + "integrity": "sha512-SrhBq4hYVjLCkBVOWaTzceJalvn5K1Zq5aQA6wXC/cYjI3frKWNPEMK3sZsJfNNQApvCQmgBcc13ZKmFj8qExw==", "dev": true, "license": "MIT", "dependencies": { - "simple-wcswidth": "^1.0.1" + "simple-wcswidth": "^1.1.2" } }, "node_modules/convert-source-map": { @@ -4205,15 +4172,15 @@ "license": "MIT" }, "node_modules/convex": { - "version": "1.25.4", - "resolved": "https://registry.npmjs.org/convex/-/convex-1.25.4.tgz", - "integrity": "sha512-LiGZZTmbe5iHWwDOYfSA00w+uDM8kgLC0ohFJW0VgQlKcs8famHCE6yuplk4wwXyj9Lhb1+yMRfrAD2ZEquqHg==", + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.29.0.tgz", + "integrity": "sha512-uoIPXRKIp2eLCkkR9WJ2vc9NtgQtx8Pml59WPUahwbrd5EuW2WLI/cf2E7XrUzOSifdQC3kJZepisk4wJNTJaA==", + "dev": true, "license": "Apache-2.0", "peer": true, "dependencies": { "esbuild": "0.25.4", - "jwt-decode": "^4.0.0", - "prettier": "3.5.3" + "prettier": "^3.0.0" }, "bin": { "convex": "bin/main.js" @@ -4285,21 +4252,6 @@ "convex": "^1.16.4" } }, - "node_modules/convex/node_modules/prettier": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", - "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", - "license": "MIT", - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, "node_modules/copy-file": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/copy-file/-/copy-file-11.1.0.tgz", @@ -4857,6 +4809,7 @@ "version": "0.25.4", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.4.tgz", "integrity": "sha512-8pgjLUcUjcgDg+2Q4NYXnPbo/vncAY4UmyaCm0jZevERqCHZIaWwdJHkf8XQtu4AxSKCdvrUbT0XUr1IdZzI8Q==", + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -6568,15 +6521,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/jwt-decode": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz", - "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -6588,9 +6532,9 @@ } }, "node_modules/langsmith": { - "version": "0.3.74", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.74.tgz", - "integrity": "sha512-ZuW3Qawz8w88XcuCRH91yTp6lsdGuwzRqZ5J0Hf5q/AjMz7DwcSv0MkE6V5W+8hFMI850QZN2Wlxwm3R9lHlZg==", + "version": "0.3.79", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.79.tgz", + "integrity": "sha512-j5uiAsyy90zxlxaMuGjb7EdcL51Yx61SpKfDOI1nMPBbemGju+lf47he4e59Hp5K63CY8XWgFP42WeZ+zuIU4Q==", "dev": true, "license": "MIT", "dependencies": { @@ -7023,9 +6967,9 @@ } }, "node_modules/marked-terminal/node_modules/chalk": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", - "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", "dev": true, "license": "MIT", "engines": { @@ -7880,9 +7824,9 @@ } }, "node_modules/npm-run-all2/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", "dev": true, "license": "MIT", "engines": { @@ -8642,7 +8586,7 @@ "version": "19.2.0", "resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz", "integrity": "sha512-tmbWg6W31tQLeB5cdIBOicJDJRR2KzXsV7uSK9iNfLWQ5bIZfxuPEHp7M8wiHyHnn0DD1i7w3Zmin0FtkrwoCQ==", - "devOptional": true, + "dev": true, "license": "MIT", "peer": true, "engines": { @@ -9915,9 +9859,9 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" }, diff --git a/package.json b/package.json index d059469..2f4c1a5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "email": "support@convex.dev", "url": "https://github.com/get-convex/rag/issues" }, - "version": "0.5.4", + "version": "0.6.0", "license": "Apache-2.0", "keywords": [ "convex", @@ -22,23 +22,23 @@ "type": "module", "scripts": { "setup": "npm i && npm run dev:backend -- --once && printf 'VITE_CONVEX_SITE_URL=' >> .env.local && npx convex env get CONVEX_SITE_URL >> .env.local", - "dev": "run-p -r 'dev:backend' 'dev:frontend' 'build:watch'", - "dev:backend": "convex dev --live-component-sources --typecheck-components", + "dev": "run-p -r 'dev:*'", + "dev:backend": "convex dev --typecheck-components", "dev:frontend": "cd example && vite --clearScreen false", - "clean": "rm -rf dist tsconfig.build.tsbuildinfo", - "build": "tsc --project ./tsconfig.build.json && npm run copy:dts", - "copy:dts": "rsync -a --include='*/' --include='*.d.ts' --exclude='*' src/ dist/ || cpy 'src/**/*.d.ts' 'dist/' --parents", - "build:watch": "npx chokidar 'tsconfig*.json' 'src/**/*.ts' -i '**/*.test.ts' -c 'npm run build' --initial", - "typecheck": "tsc --noEmit && tsc -p example/convex", - "lint": "eslint src && eslint example/convex", - "all": "run-p -r 'dev:backend' 'dev:frontend' 'build:watch' 'test:watch'", + "dev:build": "chokidar 'tsconfig*.json' 'src/**/*.ts' -i '**/*.test.ts' -c 'convex codegen --component-dir ./src/component && npm run build' --initial", + "predev": "npm run dev:backend -- --until-success", + "clean": "rm -rf dist *.tsbuildinfo", + "build": "tsc --project ./tsconfig.build.json", + "typecheck": "tsc --noEmit && tsc -p example && tsc -p example/convex", + "lint": "eslint .", + "all": "run-p -r 'dev:*' 'test:watch'", "test": "vitest run --typecheck", "test:watch": "vitest --typecheck --clearScreen false", "test:debug": "vitest --inspect-brk --no-file-parallelism", "test:coverage": "vitest run --coverage --coverage.reporter=text", "prepare": "npm run build", - "alpha": "npm run clean && npm ci && run-p test lint typecheck attw && npm version prerelease --preid alpha && npm publish --tag alpha && git push --tags", - "release": "npm run clean && npm ci && run-p test lint typecheck attw && npm version patch && npm publish && git push --tags", + "alpha": "npm run clean && npm ci && run-p test lint typecheck && npm version prerelease --preid alpha && npm publish --tag alpha && git push --tags", + "release": "npm run clean && npm ci && run-p test lint typecheck && npm version patch && npm publish && git push --tags", "version": "pbcopy <<<$npm_package_version; vim CHANGELOG.md && prettier -w CHANGELOG.md && git add CHANGELOG.md" }, "files": [ @@ -48,13 +48,22 @@ "exports": { "./package.json": "./package.json", ".": { - "@convex-dev/component-source": "./src/client/index.ts", "types": "./dist/client/index.d.ts", "default": "./dist/client/index.js" }, + "./react": { + "types": "./dist/react/index.d.ts", + "default": "./dist/react/index.js" + }, "./test": "./src/test.ts", + "./_generated/component.js": { + "types": "./dist/component/_generated/component.d.ts" + }, "./convex.config": { - "@convex-dev/component-source": "./src/component/convex.config.ts", + "types": "./dist/component/convex.config.d.ts", + "default": "./dist/component/convex.config.js" + }, + "./convex.config.js": { "types": "./dist/component/convex.config.d.ts", "default": "./dist/component/convex.config.js" } @@ -63,14 +72,14 @@ "ai": "^5.0.0" }, "peerDependencies": { - "@convex-dev/workpool": "^0.2.14", - "convex": ">=1.25.0 <1.35.0", + "@convex-dev/workpool": "^0.3.0", + "convex": "^1.24.8", "convex-helpers": "^0.1.94" }, "devDependencies": { "@ai-sdk/openai": "2.0.59", "@arethetypeswrong/cli": "0.18.2", - "@convex-dev/workpool": "0.2.19", + "@convex-dev/workpool": "0.3.0", "@edge-runtime/vm": "5.0.0", "@eslint/eslintrc": "3.3.1", "@eslint/js": "9.39.0", @@ -78,7 +87,7 @@ "@tailwindcss/postcss": "4.1.16", "@tailwindcss/typography": "0.5.19", "@types/eslint-plugin-react-refresh": "0.4.0", - "@types/node": "18.19.130", + "@types/node": "20.19.24", "@types/react": "19.2.2", "@types/react-dom": "19.2.2", "@typescript-eslint/eslint-plugin": "8.46.2", @@ -87,6 +96,7 @@ "ai": "5.0.86", "autoprefixer": "10.4.21", "chokidar-cli": "3.0.0", + "convex": "1.29.0", "convex-helpers": "0.1.104", "convex-test": "0.0.38", "cpy-cli": "6.0.0", @@ -109,7 +119,6 @@ "vite": "6.4.1", "vitest": "3.2.4" }, - "main": "./dist/client/index.js", "types": "./dist/client/index.d.ts", "module": "./dist/client/index.js" } diff --git a/rename.mjs b/rename.mjs deleted file mode 100755 index 9904c29..0000000 --- a/rename.mjs +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env node - -import { readFileSync, writeFileSync, readdirSync, statSync } from "fs"; -import { join, extname, basename } from "path"; -import readline from "readline"; - -// Utility functions for case conversion -function toPascalCase(str) { - return str - .replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : "")) - .replace(/^(.)/, (char) => char.toUpperCase()); -} - -function toCamelCase(str) { - const pascal = toPascalCase(str); - if (pascal === pascal.toUpperCase()) { - return pascal.toLowerCase(); - } - return pascal.charAt(0).toLowerCase() + pascal.slice(1); -} - -function toKebabCase(str) { - return str - .replace(/([a-z])([A-Z])/g, "$1-$2") - .replace(/[-_\s]+/g, "-") - .toLowerCase(); -} - -function toSnakeCase(str) { - return str - .replace(/([a-z])([A-Z])/g, "$1_$2") - .replace(/[-_\s]+/g, "_") - .toLowerCase(); -} - -function toSpaceCase(str) { - return str - .replace(/([a-z])([A-Z])/g, "$1 $2") - .replace(/[-_]+/g, " ") - .toLowerCase(); -} - -function toTitleCase(str) { - if (str === str.toUpperCase()) { - return str; - } - return toSpaceCase(str) - .split(" ") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" "); -} - -// Function to get all files recursively, excluding specified directories -function getAllFiles(dir, excludeDirs = [".git", "node_modules", ".cursor"]) { - const files = []; - - function traverse(currentPath) { - const items = readdirSync(currentPath); - for (const item of items) { - const fullPath = join(currentPath, item); - const stats = statSync(fullPath); - - if (stats.isDirectory()) { - if (!excludeDirs.includes(item)) { - traverse(fullPath); - } - } else { - // Only process text files (skip binary files) - const ext = extname(item).toLowerCase(); - const textExtensions = [ - ".ts", - ".tsx", - ".js", - ".jsx", - ".cjs", - ".mjs", - ".json", - ".md", - ".txt", - ".yaml", - ".yml", - ".html", - ".css", - ".scss", - ".less", - ".xml", - ".config", - ]; - - if (textExtensions.includes(ext) || !ext) { - files.push(fullPath); - } - } - } - } - - traverse(dir); - return files; -} - -// Function to replace all occurrences in a file -function replaceInFile(filePath, replacements) { - try { - let content = readFileSync(filePath, "utf8"); - let hasChanges = false; - - for (const [oldText, newText] of replacements) { - if (content.includes(oldText)) { - content = content.replaceAll(oldText, newText); - hasChanges = true; - } - } - - if (hasChanges) { - writeFileSync(filePath, content, "utf8"); - console.log(`Updated: ${filePath}`); - } - } catch (error) { - // Skip files that can't be read as text - if (error.code !== "EISDIR") { - console.warn(`Warning: Could not process ${filePath}: ${error.message}`); - } - } -} - -// Main setup function -async function setup() { - console.log("🚀 Convex Component Setup\n"); - - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - // Current directory name - const currentDirName = basename(process.cwd()); - - // Prompt for component name - const componentName = await new Promise((resolve) => { - rl.question( - `Enter your component name (e.g., "rag" or "RAG") [${currentDirName}]: `, - (answer) => { - resolve(answer.trim() || currentDirName); - } - ); - }); - - if (!componentName.trim()) { - console.error("❌ Component name is required!"); - process.exit(1); - } - - // Prompt for npm package name - const npmPackageName = await new Promise((resolve) => { - rl.question( - `Enter your npm package name [@convex-dev/${toKebabCase(componentName)}]: `, - (answer) => { - resolve(answer.trim() || `@convex-dev/${toKebabCase(componentName)}`); - } - ); - }); - - // Prompt for repository name - const repoName = await new Promise((resolve) => { - rl.question( - `Enter your repository name [get-convex/${toKebabCase(componentName)}]: `, - (answer) => { - resolve(answer.trim() || `get-convex/${toKebabCase(componentName)}`); - } - ); - }); - - rl.close(); - - // Generate all case variations - const cases = { - pascal: toPascalCase(componentName), - camel: toCamelCase(componentName), - kebab: toKebabCase(componentName), - snake: toSnakeCase(componentName), - space: toSpaceCase(componentName), - title: toTitleCase(componentName), - }; - - console.log("\n📝 Component name variations:"); - console.log(` PascalCase: ${cases.pascal}`); - console.log(` camelCase: ${cases.camel}`); - console.log(` kebab-case: ${cases.kebab}`); - console.log(` snake_case: ${cases.snake}`); - console.log(` space case: ${cases.space}`); - console.log(` Title Case: ${cases.title}`); - console.log(` NPM package: ${npmPackageName}`); - console.log(` Repository: ${repoName}\n`); - - // Define all replacements - const replacements = [ - // NPM package name - ["@convex-dev/rag", npmPackageName], - - // Repository name - ["get-convex/rag", repoName], - - // Component name variations - ["RAG", cases.pascal], - ["rag", cases.camel], - ["rag", cases.kebab], - ["rag", cases.snake], - ["rag", cases.space], - ["RAG", cases.title], - - // // Handle the component definition in convex.config.ts - // ['"rag"', `"${cases.camel}"`], - - // // Handle description (appears in package.json) - // ["A rag component for Convex.", `A ${cases.space} component for Convex.`], - ]; - - console.log("🔍 Finding files to update..."); - const files = getAllFiles("."); - console.log(`Found ${files.length} files to process.\n`); - - console.log("🔄 Processing files..."); - let processedCount = 0; - - for (const file of files) { - replaceInFile(file, replacements); - processedCount++; - } - - console.log(`\n✅ Setup complete! Processed ${processedCount} files.`); - console.log("\n📋 Next steps: check out README.md"); - - // Prompt to delete rename.mjs - const rl2 = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - const shouldDelete = await new Promise((resolve) => { - rl2.question( - "\n🗑️ Would you like to delete the rename.mjs file now? (y/N): ", - (answer) => { - resolve( - answer.toLowerCase().trim() === "y" || - answer.toLowerCase().trim() === "yes" - ); - } - ); - }); - - rl2.close(); - - if (shouldDelete) { - try { - const { unlinkSync } = await import("fs"); - unlinkSync("./rename.mjs"); - console.log("✅ rename.mjs has been deleted."); - } catch (error) { - console.error("❌ Failed to delete rename.mjs:", error.message); - } - } else { - console.log("📝 rename.mjs kept. You can delete it manually when ready."); - } -} - -// Run the setup -setup().catch(console.error); diff --git a/src/client/defaultChunker.test.ts b/src/client/defaultChunker.test.ts index 74ff811..5fb3710 100644 --- a/src/client/defaultChunker.test.ts +++ b/src/client/defaultChunker.test.ts @@ -47,7 +47,7 @@ Short para 4.`; }); expect(chunks.length).toBe(1); expect(chunks[0]).toBe( - "Short para 1.\n\nShort para 2.\n\nShort para 3.\n\nShort para 4." + "Short para 1.\n\nShort para 2.\n\nShort para 3.\n\nShort para 4.", ); expect(chunks.join("\n")).toBe(text); }); diff --git a/src/client/defaultChunker.ts b/src/client/defaultChunker.ts index 11d1a4e..0ada310 100644 --- a/src/client/defaultChunker.ts +++ b/src/client/defaultChunker.ts @@ -18,7 +18,7 @@ export function defaultChunker( maxCharsSoftLimit?: number; maxCharsHardLimit?: number; delimiter?: string; - } = {} + } = {}, ): string[] { if (!text) return []; @@ -42,7 +42,7 @@ export function defaultChunker( const processedChunk = processChunkForOutput( currentChunk, lines, - i - currentChunk.length + i - currentChunk.length, ); if (processedChunk.trim()) { chunks.push(processedChunk); @@ -73,7 +73,7 @@ export function defaultChunker( const processedChunk = processChunkForOutput( currentChunk, lines, - i - currentChunk.length + i - currentChunk.length, ); if (processedChunk.trim()) { chunks.push(processedChunk); @@ -113,7 +113,7 @@ export function defaultChunker( const processedChunk = processChunkForOutput( currentChunk, lines, - i - currentChunk.length + i - currentChunk.length, ); if (processedChunk.trim()) { chunks.push(processedChunk); @@ -134,7 +134,7 @@ export function defaultChunker( const processedChunk = processChunkForOutput( currentChunk, lines, - lines.length - currentChunk.length + lines.length - currentChunk.length, ); if (processedChunk.trim()) { chunks.push(processedChunk); @@ -149,7 +149,7 @@ export function defaultChunker( function processChunkForOutput( chunkLines: string[], allLines: string[], - startIndex: number + startIndex: number, ): string { if (chunkLines.length === 0) return ""; @@ -195,7 +195,7 @@ function maybeSplitLine(line: string, maxCharsHardLimit: number): string[] { function shouldStartNewSection( lines: string[], index: number, - delimiter: string + delimiter: string, ): boolean { if (index === 0) return false; diff --git a/src/client/fileUtils.ts b/src/client/fileUtils.ts index 47bf39d..d0bc7c1 100644 --- a/src/client/fileUtils.ts +++ b/src/client/fileUtils.ts @@ -1,5 +1,5 @@ export function guessMimeTypeFromExtension( - filename: string + filename: string, ): string | undefined { const extension = filename.split(".").pop(); if (!extension || extension.includes(" ")) { @@ -140,10 +140,10 @@ export function guessMimeTypeFromContents(buf: ArrayBuffer | string): string { */ export async function contentHashFromArrayBuffer( buffer: ArrayBuffer, - algorithm: "SHA-256" | "SHA-1" = "SHA-256" + algorithm: "SHA-256" | "SHA-1" = "SHA-256", ) { return Array.from( - new Uint8Array(await crypto.subtle.digest(algorithm, buffer)) + new Uint8Array(await crypto.subtle.digest(algorithm, buffer)), ) .map((b) => b.toString(16).padStart(2, "0")) .join(""); diff --git a/src/client/hybridRank.ts b/src/client/hybridRank.ts index 8b3425a..11c09fa 100644 --- a/src/client/hybridRank.ts +++ b/src/client/hybridRank.ts @@ -21,7 +21,7 @@ export function hybridRank( * The cutoff score for a result to be returned. */ cutoffScore?: number; - } + }, ): T[] { const k = opts?.k ?? 10; const scores: Map = new Map(); diff --git a/src/client/index.test.ts b/src/client/index.test.ts index 0c442e8..d4397e5 100644 --- a/src/client/index.test.ts +++ b/src/client/index.test.ts @@ -51,7 +51,7 @@ export const add = mutation({ text: v.string(), metadata: v.record(v.string(), v.any()), embedding: v.array(v.number()), - }) + }), ), namespace: v.string(), title: v.optional(v.string()), @@ -69,9 +69,9 @@ export const add = mutation({ v.object({ name: v.literal("customObject"), value: v.record(v.string(), v.any()), - }) - ) - ) + }), + ), + ), ), importance: v.optional(v.number()), contentHash: v.optional(v.string()), @@ -90,7 +90,7 @@ export const search = action({ v.object({ before: v.number(), after: v.number(), - }) + }), ), }, handler: async (ctx, args) => { @@ -116,12 +116,11 @@ const testApi: ApiFromModules<{ add: typeof add; search: typeof search; }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any }>["fns"] = anyApi["index.test"] as any; function dummyEmbeddings(text: string) { return Array.from({ length: 1536 }, (_, i) => - i === 0 ? text.charCodeAt(0) / 256 : 0.1 + i === 0 ? text.charCodeAt(0) / 256 : 0.1, ); } @@ -177,14 +176,15 @@ describe("RAG thick client", () => { expect(entryId).toBeDefined(); expect(status).toBe("ready"); expect(usage).toEqual({ tokens: 0 }); - const { entryId: entryId2, status: status2, usage: usage2 } = await t.mutation( - testApi.add, - { - key: "test", - chunks: [{ text: "A", metadata: {}, embedding: dummyEmbeddings("A") }], - namespace: "test", - } - ); + const { + entryId: entryId2, + status: status2, + usage: usage2, + } = await t.mutation(testApi.add, { + key: "test", + chunks: [{ text: "A", metadata: {}, embedding: dummyEmbeddings("A") }], + namespace: "test", + }); expect(entryId2).toBeDefined(); expect(status2).toBe("ready"); expect(usage2).toEqual({ tokens: 0 }); @@ -238,12 +238,12 @@ describe("RAG thick client", () => { expect(text).toContain("## Test Document:"); expect(entries).toHaveLength(1); expect(entries[0].text).toBe( - "Chunk 1 content\nChunk 2 content\nChunk 3 content" + "Chunk 1 content\nChunk 2 content\nChunk 3 content", ); // Overall text should be: "## Test Document:\nChunk 1 content\nChunk 2 content\nChunk 3 content" expect(text).toBe( - "## Test Document:\n\nChunk 1 content\nChunk 2 content\nChunk 3 content" + "## Test Document:\n\nChunk 1 content\nChunk 2 content\nChunk 3 content", ); expect(usage).toEqual({ tokens: 0 }); }); @@ -508,7 +508,7 @@ Chunk 2 contents ## Title 2: Chunk 3 contents -Chunk 4 contents` +Chunk 4 contents`, ); }); }); diff --git a/src/client/index.ts b/src/client/index.ts index fd3af79..eb69947 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -2,13 +2,14 @@ import { embed, embedMany, generateText, - type ModelMessage, type EmbeddingModel, type EmbeddingModelUsage, + type ModelMessage, } from "ai"; import { assert } from "convex-helpers"; import { createFunctionHandle, + FunctionReference, internalActionGeneric, internalMutationGeneric, type FunctionArgs, @@ -23,6 +24,8 @@ import { type RegisteredMutation, } from "convex/server"; import { type Value } from "convex/values"; +import { ComponentApi } from "../component/_generated/component.js"; +import type { NamedFilter } from "../component/filters.js"; import { CHUNK_BATCH_SIZE, filterNamesContain, @@ -32,28 +35,19 @@ import { vNamespaceId, vOnCompleteArgs, type Chunk, + type ChunkerAction, type CreateChunkArgs, type Entry, type EntryFilter, type EntryId, type Namespace, type NamespaceId, + type OnComplete, + type OnCompleteNamespace, type SearchEntry, type SearchResult, type Status, } from "../shared.js"; -import { - type RAGComponent, - type RunActionCtx, - type RunMutationCtx, - type RunQueryCtx, -} from "./types.js"; -import { - type ChunkerAction, - type OnComplete, - type OnCompleteNamespace, -} from "../shared.js"; -import type { NamedFilter } from "../component/filters.js"; import { defaultChunker } from "./defaultChunker.js"; export { hybridRank } from "./hybridRank.js"; @@ -62,7 +56,6 @@ export type { ChunkerAction, Entry, EntryId, - RAGComponent, NamespaceId, OnComplete, OnCompleteNamespace, @@ -72,18 +65,18 @@ export type { }; export { - type VEntry, - type VSearchEntry, - type EntryFilter, vEntry, + vOnCompleteArgs, vSearchEntry, vSearchResult, - vOnCompleteArgs, + type EntryFilter, + type VEntry, + type VSearchEntry, } from "../shared.js"; export { contentHashFromArrayBuffer, - guessMimeTypeFromExtension, guessMimeTypeFromContents, + guessMimeTypeFromExtension, } from "./fileUtils.js"; const DEFAULT_SEARCH_LIMIT = 10; @@ -116,12 +109,12 @@ export class RAG< * and then entry results will have the metadata type `{ source: "website" }`. */ constructor( - public component: RAGComponent, + public component: ComponentApi, public options: { embeddingDimension: number; textEmbeddingModel: EmbeddingModel; filterNames?: FilterNames; - } + }, ) {} /** @@ -135,7 +128,7 @@ export class RAG< * The filterValues you provide can be used later to search for it. */ async add( - ctx: RunMutationCtx, + ctx: CtxWith<"runMutation">, args: NamespaceSelection & EntryArgs & ( @@ -159,7 +152,7 @@ export class RAG< /** @deprecated You cannot specify both chunks and text currently. */ chunks?: undefined; } - ) + ), ): Promise<{ entryId: EntryId; status: Status; @@ -186,7 +179,7 @@ export class RAG< if (Array.isArray(chunks) && chunks.length < CHUNK_BATCH_SIZE) { const result = await createChunkArgsBatch( this.options.textEmbeddingModel, - chunks + chunks, ); allChunks = result.chunks; totalUsage.tokens += result.usage.tokens; @@ -209,7 +202,7 @@ export class RAG< }, onComplete, allChunks, - } + }, ); if (status === "ready") { return { @@ -231,7 +224,7 @@ export class RAG< for await (const batch of batchIterator(chunks, CHUNK_BATCH_SIZE)) { const result = await createChunkArgsBatch( this.options.textEmbeddingModel, - batch + batch, ); totalUsage.tokens += result.usage.tokens; const { status } = await ctx.runMutation(this.component.chunks.insert, { @@ -251,7 +244,7 @@ export class RAG< while (true) { const { status, nextStartOrder } = await ctx.runMutation( this.component.chunks.replaceChunksPage, - { entryId, startOrder } + { entryId, startOrder }, ); if (status === "ready") { break; @@ -269,7 +262,7 @@ export class RAG< } const promoted = await ctx.runMutation( this.component.entries.promoteToReady, - { entryId } + { entryId }, ); return { entryId: entryId as EntryId, @@ -307,7 +300,7 @@ export class RAG< * ``` */ async addAsync( - ctx: RunMutationCtx, + ctx: CtxWith<"runMutation">, args: NamespaceSelection & EntryArgs & { /** @@ -325,7 +318,7 @@ export class RAG< * }); */ chunkerAction: ChunkerAction; - } + }, ): Promise<{ entryId: EntryId; status: "ready" | "pending" }> { let namespaceId: NamespaceId; if ("namespaceId" in args) { @@ -359,7 +352,7 @@ export class RAG< }, onComplete, chunker, - } + }, ); return { entryId: entryId as EntryId, status }; } @@ -370,7 +363,7 @@ export class RAG< * parameters to filter and constrain the results. */ async search( - ctx: RunActionCtx, + ctx: CtxWith<"runAction">, args: { /** * The namespace to search in. e.g. a userId if entries are per-user. @@ -382,7 +375,7 @@ export class RAG< * The query to search for. Optional if embedding is provided. */ query: string | Array; - } & SearchOptions + } & SearchOptions, ): Promise<{ results: SearchResult[]; text: string; @@ -416,7 +409,7 @@ export class RAG< limit, vectorScoreThreshold, chunkContext, - } + }, ); const entriesWithTexts = entries.map((e) => { const ranges = results @@ -459,7 +452,7 @@ export class RAG< * extra context / conversation history. */ async generateText( - ctx: RunActionCtx, + ctx: CtxWith<"runAction">, args: { /** * The search options to use for context search, including the namespace. @@ -486,7 +479,7 @@ export class RAG< * to the prompt, in which case it will precede the prompt. */ messages?: ModelMessage[]; - } & Parameters[0] + } & Parameters[0], ): Promise< Awaited> & { context: { @@ -531,7 +524,7 @@ export class RAG< .map((e) => e.title ? `${e.text}` - : `${e.text}` + : `${e.text}`, ) .join("\n"); contextFooter = ""; @@ -577,12 +570,12 @@ export class RAG< * List all entries in a namespace. */ async list( - ctx: RunQueryCtx, + ctx: CtxWith<"runQuery">, args: { namespaceId?: NamespaceId; order?: "desc" | "asc"; status?: Status; - } & ({ paginationOpts: PaginationOptions } | { limit: number }) + } & ({ paginationOpts: PaginationOptions } | { limit: number }), ): Promise>> { const paginationOpts = "paginationOpts" in args @@ -601,10 +594,10 @@ export class RAG< * Get entry metadata by its id. */ async getEntry( - ctx: RunQueryCtx, + ctx: CtxWith<"runQuery">, args: { entryId: EntryId; - } + }, ): Promise | null> { const entry = await ctx.runQuery(this.component.entries.get, { entryId: args.entryId, @@ -618,13 +611,13 @@ export class RAG< * when updating content. */ async findEntryByContentHash( - ctx: RunQueryCtx, + ctx: CtxWith<"runQuery">, args: { namespace: string; key: string; /** The hash of the entry contents to try to match. */ contentHash: string; - } + }, ): Promise | null> { const entry = await ctx.runQuery(this.component.entries.findByContentHash, { namespace: args.namespace, @@ -642,7 +635,7 @@ export class RAG< * filterNames of the RAG instance. If it doesn't exist, it will be created. */ async getOrCreateNamespace( - ctx: RunMutationCtx, + ctx: CtxWith<"runMutation">, args: { /** * The namespace to get or create. e.g. a userId if entries are per-user. @@ -658,7 +651,7 @@ export class RAG< * along the way. */ onComplete?: OnCompleteNamespace; - } + }, ): Promise<{ namespaceId: NamespaceId; status: "pending" | "ready"; @@ -668,7 +661,7 @@ export class RAG< : undefined; assert( !onComplete || args.status === "pending", - "You can only supply an onComplete handler for pending namespaces" + "You can only supply an onComplete handler for pending namespaces", ); const { namespaceId, status } = await ctx.runMutation( this.component.namespaces.getOrCreate, @@ -679,7 +672,7 @@ export class RAG< modelId: getModelId(this.options.textEmbeddingModel), dimension: this.options.embeddingDimension, filterNames: this.options.filterNames ?? [], - } + }, ); return { namespaceId: namespaceId as NamespaceId, status }; } @@ -689,10 +682,10 @@ export class RAG< * filterNames of the RAG instance. If it doesn't exist, it will return null. */ async getNamespace( - ctx: RunQueryCtx, + ctx: CtxWith<"runQuery">, args: { namespace: string; - } + }, ): Promise { return ctx.runQuery(this.component.namespaces.get, { namespace: args.namespace, @@ -706,12 +699,12 @@ export class RAG< * List all chunks for an entry, paginated. */ async listChunks( - ctx: RunQueryCtx, + ctx: CtxWith<"runQuery">, args: { paginationOpts: PaginationOptions; entryId: EntryId; order?: "desc" | "asc"; - } + }, ): Promise> { return ctx.runQuery(this.component.chunks.list, { entryId: args.entryId, @@ -723,7 +716,7 @@ export class RAG< /** * Delete an entry and all its chunks in the background using a workpool. */ - async deleteAsync(ctx: RunMutationCtx, args: { entryId: EntryId }) { + async deleteAsync(ctx: CtxWith<"runMutation">, args: { entryId: EntryId }) { await ctx.runMutation(this.component.entries.deleteAsync, { entryId: args.entryId, startOrder: 0, @@ -736,17 +729,26 @@ export class RAG< * you're likely running this in a mutation. * Use `deleteAsync` or run `delete` in an action. */ - async delete(ctx: RunActionCtx, args: { entryId: EntryId }): Promise; + async delete( + ctx: CtxWith<"runAction">, + args: { entryId: EntryId }, + ): Promise; /** @deprecated Use `deleteAsync` in mutations. */ - async delete(ctx: RunMutationCtx, args: { entryId: EntryId }): Promise; - async delete(ctx: RunActionCtx | RunMutationCtx, args: { entryId: EntryId }) { + async delete( + ctx: CtxWith<"runMutation">, + args: { entryId: EntryId }, + ): Promise; + async delete( + ctx: CtxWith<"runMutation"> | CtxWith<"runAction">, + args: { entryId: EntryId }, + ) { if ("runAction" in ctx) { await ctx.runAction(this.component.entries.deleteSync, { entryId: args.entryId, }); } else { console.warn( - "You are running `rag.delete` in a mutation. This is deprecated. Use `rag.deleteAsync` from mutations, or `rag.delete` in actions." + "You are running `rag.delete` in a mutation. This is deprecated. Use `rag.deleteAsync` from mutations, or `rag.delete` in actions.", ); await ctx.runMutation(this.component.entries.deleteAsync, { entryId: args.entryId, @@ -759,8 +761,8 @@ export class RAG< * Delete all entries with a given key (asynchrounously). */ async deleteByKeyAsync( - ctx: RunMutationCtx, - args: { namespaceId: NamespaceId; key: string; beforeVersion?: number } + ctx: CtxWith<"runMutation">, + args: { namespaceId: NamespaceId; key: string; beforeVersion?: number }, ) { await ctx.runMutation(this.component.entries.deleteByKeyAsync, { namespaceId: args.namespaceId, @@ -776,8 +778,8 @@ export class RAG< * Use `deleteByKeyAsync` or run `delete` in an action. */ async deleteByKey( - ctx: RunActionCtx, - args: { namespaceId: NamespaceId; key: string; beforeVersion?: number } + ctx: CtxWith<"runAction">, + args: { namespaceId: NamespaceId; key: string; beforeVersion?: number }, ) { await ctx.runAction(this.component.entries.deleteByKeySync, args); } @@ -803,8 +805,8 @@ export class RAG< defineOnComplete( fn: ( ctx: GenericMutationCtx, - args: OnCompleteArgs - ) => Promise + args: OnCompleteArgs, + ) => Promise, ): RegisteredMutation<"internal", FunctionArgs, null> { return internalMutationGeneric({ args: vOnCompleteArgs, @@ -833,8 +835,11 @@ export class RAG< defineChunkerAction( fn: ( ctx: GenericActionCtx, - args: { namespace: Namespace; entry: Entry } - ) => AsyncIterable | Promise<{ chunks: InputChunk[] }> + args: { + namespace: Namespace; + entry: Entry; + }, + ) => AsyncIterable | Promise<{ chunks: InputChunk[] }>, ): RegisteredAction< "internal", FunctionArgs, @@ -848,26 +853,26 @@ export class RAG< if (namespace.modelId !== modelId) { console.error( `You are using a different embedding model ${modelId} for asynchronously ` + - `generating chunks than the one provided when it was started: ${namespace.modelId}` + `generating chunks than the one provided when it was started: ${namespace.modelId}`, ); return; } if (namespace.dimension !== this.options.embeddingDimension) { console.error( `You are using a different embedding dimension ${this.options.embeddingDimension} for asynchronously ` + - `generating chunks than the one provided when it was started: ${namespace.dimension}` + `generating chunks than the one provided when it was started: ${namespace.dimension}`, ); return; } if ( !filterNamesContain( namespace.filterNames, - this.options.filterNames ?? [] + this.options.filterNames ?? [], ) ) { console.error( `You are using a different filters (${this.options.filterNames?.join(", ")}) for asynchronously ` + - `generating chunks than the one provided when it was started: ${namespace.filterNames.join(", ")}` + `generating chunks than the one provided when it was started: ${namespace.filterNames.join(", ")}`, ); return; } @@ -889,23 +894,23 @@ export class RAG< let batchOrder = 0; for await (const batch of batchIterator( chunkIterator, - CHUNK_BATCH_SIZE + CHUNK_BATCH_SIZE, )) { const result = await createChunkArgsBatch( this.options.textEmbeddingModel, - batch + batch, ); await ctx.runMutation( args.insertChunks as FunctionHandle< "mutation", - FunctionArgs, + FunctionArgs, null >, { entryId: entry.entryId, startOrder: batchOrder, chunks: result.chunks, - } + }, ); batchOrder += result.chunks.length; } @@ -916,7 +921,7 @@ export class RAG< async function* batchIterator( iterator: Iterable | AsyncIterable, - batchSize: number + batchSize: number, ): AsyncIterable { let batch: T[] = []; for await (const item of iterator) { @@ -933,21 +938,21 @@ async function* batchIterator( function validateAddFilterValues( filterValues: NamedFilter[] | undefined, - filterNames: string[] | undefined + filterNames: string[] | undefined, ) { if (!filterValues) { return; } if (!filterNames) { throw new Error( - "You must provide filter names to RAG to add entries with filters." + "You must provide filter names to RAG to add entries with filters.", ); } const seen = new Set(); for (const filterValue of filterValues) { if (seen.has(filterValue.name)) { throw new Error( - `You cannot provide the same filter name twice: ${filterValue.name}.` + `You cannot provide the same filter name twice: ${filterValue.name}.`, ); } seen.add(filterValue.name); @@ -955,7 +960,7 @@ function validateAddFilterValues( for (const filterName of filterNames) { if (!seen.has(filterName)) { throw new Error( - `Filter name ${filterName} is not valid (one of ${filterNames.join(", ")}).` + `Filter name ${filterName} is not valid (one of ${filterNames.join(", ")}).`, ); } } @@ -971,7 +976,7 @@ function makeBatches(items: T[], batchSize: number): T[][] { async function createChunkArgsBatch( embedModel: EmbeddingModel, - chunks: InputChunk[] + chunks: InputChunk[], ): Promise<{ chunks: CreateChunkArgs[]; usage: EmbeddingModelUsage }> { const argsMaybeMissingEmbeddings: (Omit & { embedding?: number[]; @@ -1003,7 +1008,7 @@ async function createChunkArgsBatch( : { text: arg.content.text, index, - } + }, ) .filter((b) => b !== null); const totalUsage: EmbeddingModelUsage = { tokens: 0 }; @@ -1213,3 +1218,21 @@ export function getProviderName(embeddingModel: ModelOrMetadata): string { } return embeddingModel.provider; } + +type CtxWith = Pick< + { + runQuery: >( + query: Query, + args: FunctionArgs, + ) => Promise>; + runMutation: >( + mutation: Mutation, + args: FunctionArgs, + ) => Promise>; + runAction: >( + action: Action, + args: FunctionArgs, + ) => Promise>; + }, + T +>; diff --git a/src/client/setup.test.ts b/src/client/setup.test.ts index 440b45d..eaa8a46 100644 --- a/src/client/setup.test.ts +++ b/src/client/setup.test.ts @@ -8,7 +8,7 @@ import { type GenericSchema, type SchemaDefinition, } from "convex/server"; -import { type RAGComponent } from "./index.js"; +import { type ComponentApi } from "../component/_generated/component.js"; import { componentsGeneric } from "convex/server"; import componentSchema from "../component/schema.js"; export { componentSchema }; @@ -22,7 +22,7 @@ export function initConvexTest< return t; } export const components = componentsGeneric() as unknown as { - rag: RAGComponent; + rag: ComponentApi; }; test("setup", () => {}); diff --git a/src/client/types.ts b/src/client/types.ts deleted file mode 100644 index 75f8f00..0000000 --- a/src/client/types.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { - Expand, - FunctionReference, - GenericActionCtx, - GenericDataModel, - GenericMutationCtx, - GenericQueryCtx, - StorageActionWriter, - StorageReader, -} from "convex/server"; -import { type GenericId } from "convex/values"; -import type { api } from "../component/_generated/api.js"; - -// UseApi is an alternative that has jump-to-definition but is -// less stable and reliant on types within the component files, which can cause -// issues where passing `components.foo` doesn't match the argument -export type RAGComponent = UseApi; - -// Type utils follow - -export type RunQueryCtx = { - runQuery: GenericQueryCtx["runQuery"]; -}; -export type RunMutationCtx = { - runQuery: GenericMutationCtx["runQuery"]; - runMutation: GenericMutationCtx["runMutation"]; -}; -export type RunActionCtx = { - runQuery: GenericActionCtx["runQuery"]; - runMutation: GenericActionCtx["runMutation"]; - runAction: GenericActionCtx["runAction"]; -}; -export type ActionCtx = RunActionCtx & { - storage: StorageActionWriter; -}; -export type QueryCtx = RunQueryCtx & { - storage: StorageReader; -}; - -export type OpaqueIds = - T extends GenericId - ? string - : T extends string - ? `${T}` extends T - ? T - : string - : T extends (infer U)[] - ? OpaqueIds[] - : T extends ArrayBuffer - ? ArrayBuffer - : T extends object - ? { [K in keyof T]: OpaqueIds } - : T; - -export type UseApi = Expand<{ - [mod in keyof API]: API[mod] extends FunctionReference< - infer FType, - "public", - infer FArgs, - infer FReturnType, - infer FComponentPath - > - ? FunctionReference< - FType, - "internal", - OpaqueIds, - OpaqueIds, - FComponentPath - > - : UseApi; -}>; diff --git a/src/component/_generated/api.d.ts b/src/component/_generated/api.d.ts deleted file mode 100644 index fc268d2..0000000 --- a/src/component/_generated/api.d.ts +++ /dev/null @@ -1,631 +0,0 @@ -/* eslint-disable */ -/** - * Generated `api` utility. - * - * THIS CODE IS AUTOMATICALLY GENERATED. - * - * To regenerate, run `npx convex dev`. - * @module - */ - -import type { FunctionReference } from "convex/server"; -import type { GenericId as Id } from "convex/values"; - -/** - * A utility for referencing Convex functions in your app's public API. - * - * Usage: - * ```js - * const myFunctionReference = api.myModule.myFunction; - * ``` - */ -export declare const api: { - chunks: { - insert: FunctionReference< - "mutation", - "public", - { - chunks: Array<{ - content: { metadata?: Record; text: string }; - embedding: Array; - searchableText?: string; - }>; - entryId: Id<"entries">; - startOrder: number; - }, - { status: "pending" | "ready" | "replaced" } - >; - list: FunctionReference< - "query", - "public", - { - entryId: Id<"entries">; - order: "desc" | "asc"; - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - }, - { - continueCursor: string; - isDone: boolean; - page: Array<{ - metadata?: Record; - order: number; - state: "pending" | "ready" | "replaced"; - text: string; - }>; - pageStatus?: "SplitRecommended" | "SplitRequired" | null; - splitCursor?: string | null; - } - >; - replaceChunksPage: FunctionReference< - "mutation", - "public", - { entryId: Id<"entries">; startOrder: number }, - { nextStartOrder: number; status: "pending" | "ready" | "replaced" } - >; - }; - entries: { - add: FunctionReference< - "mutation", - "public", - { - allChunks?: Array<{ - content: { metadata?: Record; text: string }; - embedding: Array; - searchableText?: string; - }>; - entry: { - contentHash?: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - namespaceId: Id<"namespaces">; - title?: string; - }; - onComplete?: string; - }, - { - created: boolean; - entryId: Id<"entries">; - status: "pending" | "ready" | "replaced"; - } - >; - addAsync: FunctionReference< - "mutation", - "public", - { - chunker: string; - entry: { - contentHash?: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - namespaceId: Id<"namespaces">; - title?: string; - }; - onComplete?: string; - }, - { created: boolean; entryId: Id<"entries">; status: "pending" | "ready" } - >; - deleteAsync: FunctionReference< - "mutation", - "public", - { entryId: Id<"entries">; startOrder: number }, - null - >; - deleteByKeyAsync: FunctionReference< - "mutation", - "public", - { beforeVersion?: number; key: string; namespaceId: Id<"namespaces"> }, - null - >; - deleteByKeySync: FunctionReference< - "action", - "public", - { key: string; namespaceId: Id<"namespaces"> }, - null - >; - deleteSync: FunctionReference< - "action", - "public", - { entryId: Id<"entries"> }, - null - >; - findByContentHash: FunctionReference< - "query", - "public", - { - contentHash: string; - dimension: number; - filterNames: Array; - key: string; - modelId: string; - namespace: string; - }, - { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - } | null - >; - get: FunctionReference< - "query", - "public", - { entryId: Id<"entries"> }, - { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - } | null - >; - list: FunctionReference< - "query", - "public", - { - namespaceId?: Id<"namespaces">; - order?: "desc" | "asc"; - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - status: "pending" | "ready" | "replaced"; - }, - { - continueCursor: string; - isDone: boolean; - page: Array<{ - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }>; - pageStatus?: "SplitRecommended" | "SplitRequired" | null; - splitCursor?: string | null; - } - >; - promoteToReady: FunctionReference< - "mutation", - "public", - { entryId: Id<"entries"> }, - { - replacedEntry: { - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - } | null; - } - >; - }; - namespaces: { - deleteNamespace: FunctionReference< - "mutation", - "public", - { namespaceId: Id<"namespaces"> }, - { - deletedNamespace: null | { - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }; - } - >; - deleteNamespaceSync: FunctionReference< - "action", - "public", - { namespaceId: Id<"namespaces"> }, - null - >; - get: FunctionReference< - "query", - "public", - { - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - }, - null | { - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - } - >; - getOrCreate: FunctionReference< - "mutation", - "public", - { - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - onComplete?: string; - status: "pending" | "ready"; - }, - { namespaceId: Id<"namespaces">; status: "pending" | "ready" } - >; - list: FunctionReference< - "query", - "public", - { - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - status: "pending" | "ready" | "replaced"; - }, - { - continueCursor: string; - isDone: boolean; - page: Array<{ - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }>; - pageStatus?: "SplitRecommended" | "SplitRequired" | null; - splitCursor?: string | null; - } - >; - listNamespaceVersions: FunctionReference< - "query", - "public", - { - namespace: string; - paginationOpts: { - cursor: string | null; - endCursor?: string | null; - id?: number; - maximumBytesRead?: number; - maximumRowsRead?: number; - numItems: number; - }; - }, - { - continueCursor: string; - isDone: boolean; - page: Array<{ - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }>; - pageStatus?: "SplitRecommended" | "SplitRequired" | null; - splitCursor?: string | null; - } - >; - lookup: FunctionReference< - "query", - "public", - { - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - }, - null | Id<"namespaces"> - >; - promoteToReady: FunctionReference< - "mutation", - "public", - { namespaceId: Id<"namespaces"> }, - { - replacedNamespace: null | { - createdAt: number; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - namespaceId: string; - status: "pending" | "ready" | "replaced"; - version: number; - }; - } - >; - }; - search: { - search: FunctionReference< - "action", - "public", - { - chunkContext?: { after: number; before: number }; - embedding: Array; - filters: Array<{ name: string; value: any }>; - limit: number; - modelId: string; - namespace: string; - vectorScoreThreshold?: number; - }, - { - entries: Array<{ - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }>; - results: Array<{ - content: Array<{ metadata?: Record; text: string }>; - entryId: string; - order: number; - score: number; - startOrder: number; - }>; - } - >; - }; -}; - -/** - * A utility for referencing Convex functions in your app's internal API. - * - * Usage: - * ```js - * const myFunctionReference = internal.myModule.myFunction; - * ``` - */ -export declare const internal: { - chunks: { - deleteChunksPage: FunctionReference< - "mutation", - "internal", - { entryId: Id<"entries">; startOrder: number }, - { isDone: boolean; nextStartOrder: number } - >; - getRangesOfChunks: FunctionReference< - "query", - "internal", - { - chunkContext: { after: number; before: number }; - embeddingIds: Array< - | Id<"vectors_128"> - | Id<"vectors_256"> - | Id<"vectors_512"> - | Id<"vectors_768"> - | Id<"vectors_1024"> - | Id<"vectors_1408"> - | Id<"vectors_1536"> - | Id<"vectors_2048"> - | Id<"vectors_3072"> - | Id<"vectors_4096"> - >; - }, - { - entries: Array<{ - contentHash?: string; - entryId: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - replacedAt?: number; - status: "pending" | "ready" | "replaced"; - title?: string; - }>; - ranges: Array; text: string }>; - entryId: Id<"entries">; - order: number; - startOrder: number; - }>; - } - >; - }; - entries: { - _del: FunctionReference< - "mutation", - "internal", - { entryId: Id<"entries"> }, - null - >; - addAsyncOnComplete: FunctionReference< - "mutation", - "internal", - { - context: Id<"entries">; - result: - | { kind: "success"; returnValue: any } - | { error: string; kind: "failed" } - | { kind: "canceled" }; - workId: string; - }, - null - >; - getEntriesForNamespaceByKey: FunctionReference< - "query", - "internal", - { beforeVersion?: number; key: string; namespaceId: Id<"namespaces"> }, - Array<{ - _creationTime: number; - _id: Id<"entries">; - contentHash?: string; - filterValues: Array<{ name: string; value: any }>; - importance: number; - key?: string; - metadata?: Record; - namespaceId: Id<"namespaces">; - status: - | { kind: "pending"; onComplete?: string } - | { kind: "ready" } - | { kind: "replaced"; replacedAt: number }; - title?: string; - version: number; - }> - >; - }; - namespaces: { - getCompatibleNamespace: FunctionReference< - "query", - "internal", - { - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - }, - null | { - _creationTime: number; - _id: Id<"namespaces">; - dimension: number; - filterNames: Array; - modelId: string; - namespace: string; - status: - | { kind: "pending"; onComplete?: string } - | { kind: "ready" } - | { kind: "replaced"; replacedAt: number }; - version: number; - } - >; - }; -}; - -export declare const components: { - workpool: { - lib: { - cancel: FunctionReference< - "mutation", - "internal", - { - id: string; - logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; - }, - any - >; - cancelAll: FunctionReference< - "mutation", - "internal", - { - before?: number; - limit?: number; - logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; - }, - any - >; - enqueue: FunctionReference< - "mutation", - "internal", - { - config: { - logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; - maxParallelism: number; - }; - fnArgs: any; - fnHandle: string; - fnName: string; - fnType: "action" | "mutation" | "query"; - onComplete?: { context?: any; fnHandle: string }; - retryBehavior?: { - base: number; - initialBackoffMs: number; - maxAttempts: number; - }; - runAt: number; - }, - string - >; - enqueueBatch: FunctionReference< - "mutation", - "internal", - { - config: { - logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; - maxParallelism: number; - }; - items: Array<{ - fnArgs: any; - fnHandle: string; - fnName: string; - fnType: "action" | "mutation" | "query"; - onComplete?: { context?: any; fnHandle: string }; - retryBehavior?: { - base: number; - initialBackoffMs: number; - maxAttempts: number; - }; - runAt: number; - }>; - }, - Array - >; - status: FunctionReference< - "query", - "internal", - { id: string }, - | { previousAttempts: number; state: "pending" } - | { previousAttempts: number; state: "running" } - | { state: "finished" } - >; - statusBatch: FunctionReference< - "query", - "internal", - { ids: Array }, - Array< - | { previousAttempts: number; state: "pending" } - | { previousAttempts: number; state: "running" } - | { state: "finished" } - > - >; - }; - }; -}; diff --git a/src/component/_generated/api.js b/src/component/_generated/api.js deleted file mode 100644 index 44bf985..0000000 --- a/src/component/_generated/api.js +++ /dev/null @@ -1,23 +0,0 @@ -/* eslint-disable */ -/** - * Generated `api` utility. - * - * THIS CODE IS AUTOMATICALLY GENERATED. - * - * To regenerate, run `npx convex dev`. - * @module - */ - -import { anyApi, componentsGeneric } from "convex/server"; - -/** - * A utility for referencing Convex functions in your app's API. - * - * Usage: - * ```js - * const myFunctionReference = api.myModule.myFunction; - * ``` - */ -export const api = anyApi; -export const internal = anyApi; -export const components = componentsGeneric(); diff --git a/src/component/_generated/api.ts b/src/component/_generated/api.ts new file mode 100644 index 0000000..2badc28 --- /dev/null +++ b/src/component/_generated/api.ts @@ -0,0 +1,152 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type * as chunks from "../chunks.js"; +import type * as embeddings_importance from "../embeddings/importance.js"; +import type * as embeddings_index from "../embeddings/index.js"; +import type * as embeddings_tables from "../embeddings/tables.js"; +import type * as entries from "../entries.js"; +import type * as filters from "../filters.js"; +import type * as namespaces from "../namespaces.js"; +import type * as search from "../search.js"; + +import type { + ApiFromModules, + FilterApi, + FunctionReference, +} from "convex/server"; +import { anyApi, componentsGeneric } from "convex/server"; + +const fullApi: ApiFromModules<{ + chunks: typeof chunks; + "embeddings/importance": typeof embeddings_importance; + "embeddings/index": typeof embeddings_index; + "embeddings/tables": typeof embeddings_tables; + entries: typeof entries; + filters: typeof filters; + namespaces: typeof namespaces; + search: typeof search; +}> = anyApi as any; + +/** + * A utility for referencing Convex functions in your app's public API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export const api: FilterApi< + typeof fullApi, + FunctionReference +> = anyApi as any; + +/** + * A utility for referencing Convex functions in your app's internal API. + * + * Usage: + * ```js + * const myFunctionReference = internal.myModule.myFunction; + * ``` + */ +export const internal: FilterApi< + typeof fullApi, + FunctionReference +> = anyApi as any; + +export const components = componentsGeneric() as unknown as { + workpool: { + lib: { + cancel: FunctionReference< + "mutation", + "internal", + { + id: string; + logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; + }, + any + >; + cancelAll: FunctionReference< + "mutation", + "internal", + { + before?: number; + limit?: number; + logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; + }, + any + >; + enqueue: FunctionReference< + "mutation", + "internal", + { + config: { + logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; + maxParallelism: number; + }; + fnArgs: any; + fnHandle: string; + fnName: string; + fnType: "action" | "mutation" | "query"; + onComplete?: { context?: any; fnHandle: string }; + retryBehavior?: { + base: number; + initialBackoffMs: number; + maxAttempts: number; + }; + runAt: number; + }, + string + >; + enqueueBatch: FunctionReference< + "mutation", + "internal", + { + config: { + logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR"; + maxParallelism: number; + }; + items: Array<{ + fnArgs: any; + fnHandle: string; + fnName: string; + fnType: "action" | "mutation" | "query"; + onComplete?: { context?: any; fnHandle: string }; + retryBehavior?: { + base: number; + initialBackoffMs: number; + maxAttempts: number; + }; + runAt: number; + }>; + }, + Array + >; + status: FunctionReference< + "query", + "internal", + { id: string }, + | { previousAttempts: number; state: "pending" } + | { previousAttempts: number; state: "running" } + | { state: "finished" } + >; + statusBatch: FunctionReference< + "query", + "internal", + { ids: Array }, + Array< + | { previousAttempts: number; state: "pending" } + | { previousAttempts: number; state: "running" } + | { state: "finished" } + > + >; + }; + }; +}; diff --git a/src/component/_generated/component.ts b/src/component/_generated/component.ts new file mode 100644 index 0000000..338273f --- /dev/null +++ b/src/component/_generated/component.ts @@ -0,0 +1,442 @@ +/* eslint-disable */ +/** + * Generated `ComponentApi` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type { FunctionReference } from "convex/server"; + +/** + * A utility for referencing a Convex component's exposed API. + * + * Useful when expecting a parameter like `components.myComponent`. + * Usage: + * ```ts + * async function myFunction(ctx: QueryCtx, component: ComponentApi) { + * return ctx.runQuery(component.someFile.someQuery, { ...args }); + * } + * ``` + */ +export type ComponentApi = + { + chunks: { + insert: FunctionReference< + "mutation", + "internal", + { + chunks: Array<{ + content: { metadata?: Record; text: string }; + embedding: Array; + searchableText?: string; + }>; + entryId: string; + startOrder: number; + }, + { status: "pending" | "ready" | "replaced" }, + Name + >; + list: FunctionReference< + "query", + "internal", + { + entryId: string; + order: "desc" | "asc"; + paginationOpts: { + cursor: string | null; + endCursor?: string | null; + id?: number; + maximumBytesRead?: number; + maximumRowsRead?: number; + numItems: number; + }; + }, + { + continueCursor: string; + isDone: boolean; + page: Array<{ + metadata?: Record; + order: number; + state: "pending" | "ready" | "replaced"; + text: string; + }>; + pageStatus?: "SplitRecommended" | "SplitRequired" | null; + splitCursor?: string | null; + }, + Name + >; + replaceChunksPage: FunctionReference< + "mutation", + "internal", + { entryId: string; startOrder: number }, + { nextStartOrder: number; status: "pending" | "ready" | "replaced" }, + Name + >; + }; + entries: { + add: FunctionReference< + "mutation", + "internal", + { + allChunks?: Array<{ + content: { metadata?: Record; text: string }; + embedding: Array; + searchableText?: string; + }>; + entry: { + contentHash?: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + namespaceId: string; + title?: string; + }; + onComplete?: string; + }, + { + created: boolean; + entryId: string; + status: "pending" | "ready" | "replaced"; + }, + Name + >; + addAsync: FunctionReference< + "mutation", + "internal", + { + chunker: string; + entry: { + contentHash?: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + namespaceId: string; + title?: string; + }; + onComplete?: string; + }, + { created: boolean; entryId: string; status: "pending" | "ready" }, + Name + >; + deleteAsync: FunctionReference< + "mutation", + "internal", + { entryId: string; startOrder: number }, + null, + Name + >; + deleteByKeyAsync: FunctionReference< + "mutation", + "internal", + { beforeVersion?: number; key: string; namespaceId: string }, + null, + Name + >; + deleteByKeySync: FunctionReference< + "action", + "internal", + { key: string; namespaceId: string }, + null, + Name + >; + deleteSync: FunctionReference< + "action", + "internal", + { entryId: string }, + null, + Name + >; + findByContentHash: FunctionReference< + "query", + "internal", + { + contentHash: string; + dimension: number; + filterNames: Array; + key: string; + modelId: string; + namespace: string; + }, + { + contentHash?: string; + entryId: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + replacedAt?: number; + status: "pending" | "ready" | "replaced"; + title?: string; + } | null, + Name + >; + get: FunctionReference< + "query", + "internal", + { entryId: string }, + { + contentHash?: string; + entryId: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + replacedAt?: number; + status: "pending" | "ready" | "replaced"; + title?: string; + } | null, + Name + >; + list: FunctionReference< + "query", + "internal", + { + namespaceId?: string; + order?: "desc" | "asc"; + paginationOpts: { + cursor: string | null; + endCursor?: string | null; + id?: number; + maximumBytesRead?: number; + maximumRowsRead?: number; + numItems: number; + }; + status: "pending" | "ready" | "replaced"; + }, + { + continueCursor: string; + isDone: boolean; + page: Array<{ + contentHash?: string; + entryId: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + replacedAt?: number; + status: "pending" | "ready" | "replaced"; + title?: string; + }>; + pageStatus?: "SplitRecommended" | "SplitRequired" | null; + splitCursor?: string | null; + }, + Name + >; + promoteToReady: FunctionReference< + "mutation", + "internal", + { entryId: string }, + { + replacedEntry: { + contentHash?: string; + entryId: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + replacedAt?: number; + status: "pending" | "ready" | "replaced"; + title?: string; + } | null; + }, + Name + >; + }; + namespaces: { + deleteNamespace: FunctionReference< + "mutation", + "internal", + { namespaceId: string }, + { + deletedNamespace: null | { + createdAt: number; + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + namespaceId: string; + status: "pending" | "ready" | "replaced"; + version: number; + }; + }, + Name + >; + deleteNamespaceSync: FunctionReference< + "action", + "internal", + { namespaceId: string }, + null, + Name + >; + get: FunctionReference< + "query", + "internal", + { + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + }, + null | { + createdAt: number; + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + namespaceId: string; + status: "pending" | "ready" | "replaced"; + version: number; + }, + Name + >; + getOrCreate: FunctionReference< + "mutation", + "internal", + { + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + onComplete?: string; + status: "pending" | "ready"; + }, + { namespaceId: string; status: "pending" | "ready" }, + Name + >; + list: FunctionReference< + "query", + "internal", + { + paginationOpts: { + cursor: string | null; + endCursor?: string | null; + id?: number; + maximumBytesRead?: number; + maximumRowsRead?: number; + numItems: number; + }; + status: "pending" | "ready" | "replaced"; + }, + { + continueCursor: string; + isDone: boolean; + page: Array<{ + createdAt: number; + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + namespaceId: string; + status: "pending" | "ready" | "replaced"; + version: number; + }>; + pageStatus?: "SplitRecommended" | "SplitRequired" | null; + splitCursor?: string | null; + }, + Name + >; + listNamespaceVersions: FunctionReference< + "query", + "internal", + { + namespace: string; + paginationOpts: { + cursor: string | null; + endCursor?: string | null; + id?: number; + maximumBytesRead?: number; + maximumRowsRead?: number; + numItems: number; + }; + }, + { + continueCursor: string; + isDone: boolean; + page: Array<{ + createdAt: number; + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + namespaceId: string; + status: "pending" | "ready" | "replaced"; + version: number; + }>; + pageStatus?: "SplitRecommended" | "SplitRequired" | null; + splitCursor?: string | null; + }, + Name + >; + lookup: FunctionReference< + "query", + "internal", + { + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + }, + null | string, + Name + >; + promoteToReady: FunctionReference< + "mutation", + "internal", + { namespaceId: string }, + { + replacedNamespace: null | { + createdAt: number; + dimension: number; + filterNames: Array; + modelId: string; + namespace: string; + namespaceId: string; + status: "pending" | "ready" | "replaced"; + version: number; + }; + }, + Name + >; + }; + search: { + search: FunctionReference< + "action", + "internal", + { + chunkContext?: { after: number; before: number }; + embedding: Array; + filters: Array<{ name: string; value: any }>; + limit: number; + modelId: string; + namespace: string; + vectorScoreThreshold?: number; + }, + { + entries: Array<{ + contentHash?: string; + entryId: string; + filterValues: Array<{ name: string; value: any }>; + importance: number; + key?: string; + metadata?: Record; + replacedAt?: number; + status: "pending" | "ready" | "replaced"; + title?: string; + }>; + results: Array<{ + content: Array<{ metadata?: Record; text: string }>; + entryId: string; + order: number; + score: number; + startOrder: number; + }>; + }, + Name + >; + }; + }; diff --git a/src/component/_generated/dataModel.d.ts b/src/component/_generated/dataModel.ts similarity index 100% rename from src/component/_generated/dataModel.d.ts rename to src/component/_generated/dataModel.ts diff --git a/src/component/_generated/server.js b/src/component/_generated/server.js deleted file mode 100644 index 4a21df4..0000000 --- a/src/component/_generated/server.js +++ /dev/null @@ -1,90 +0,0 @@ -/* eslint-disable */ -/** - * Generated utilities for implementing server-side Convex query and mutation functions. - * - * THIS CODE IS AUTOMATICALLY GENERATED. - * - * To regenerate, run `npx convex dev`. - * @module - */ - -import { - actionGeneric, - httpActionGeneric, - queryGeneric, - mutationGeneric, - internalActionGeneric, - internalMutationGeneric, - internalQueryGeneric, - componentsGeneric, -} from "convex/server"; - -/** - * Define a query in this Convex app's public API. - * - * This function will be allowed to read your Convex database and will be accessible from the client. - * - * @param func - The query function. It receives a {@link QueryCtx} as its first argument. - * @returns The wrapped query. Include this as an `export` to name it and make it accessible. - */ -export const query = queryGeneric; - -/** - * Define a query that is only accessible from other Convex functions (but not from the client). - * - * This function will be allowed to read from your Convex database. It will not be accessible from the client. - * - * @param func - The query function. It receives a {@link QueryCtx} as its first argument. - * @returns The wrapped query. Include this as an `export` to name it and make it accessible. - */ -export const internalQuery = internalQueryGeneric; - -/** - * Define a mutation in this Convex app's public API. - * - * This function will be allowed to modify your Convex database and will be accessible from the client. - * - * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. - * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. - */ -export const mutation = mutationGeneric; - -/** - * Define a mutation that is only accessible from other Convex functions (but not from the client). - * - * This function will be allowed to modify your Convex database. It will not be accessible from the client. - * - * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. - * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. - */ -export const internalMutation = internalMutationGeneric; - -/** - * Define an action in this Convex app's public API. - * - * An action is a function which can execute any JavaScript code, including non-deterministic - * code and code with side-effects, like calling third-party services. - * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. - * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. - * - * @param func - The action. It receives an {@link ActionCtx} as its first argument. - * @returns The wrapped action. Include this as an `export` to name it and make it accessible. - */ -export const action = actionGeneric; - -/** - * Define an action that is only accessible from other Convex functions (but not from the client). - * - * @param func - The function. It receives an {@link ActionCtx} as its first argument. - * @returns The wrapped function. Include this as an `export` to name it and make it accessible. - */ -export const internalAction = internalActionGeneric; - -/** - * Define a Convex HTTP action. - * - * @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object - * as its second. - * @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. - */ -export const httpAction = httpActionGeneric; diff --git a/src/component/_generated/server.d.ts b/src/component/_generated/server.ts similarity index 79% rename from src/component/_generated/server.d.ts rename to src/component/_generated/server.ts index b5c6828..24994e4 100644 --- a/src/component/_generated/server.d.ts +++ b/src/component/_generated/server.ts @@ -8,9 +8,8 @@ * @module */ -import { +import type { ActionBuilder, - AnyComponents, HttpActionBuilder, MutationBuilder, QueryBuilder, @@ -19,15 +18,18 @@ import { GenericQueryCtx, GenericDatabaseReader, GenericDatabaseWriter, - FunctionReference, +} from "convex/server"; +import { + actionGeneric, + httpActionGeneric, + queryGeneric, + mutationGeneric, + internalActionGeneric, + internalMutationGeneric, + internalQueryGeneric, } from "convex/server"; import type { DataModel } from "./dataModel.js"; -type GenericCtx = - | GenericActionCtx - | GenericMutationCtx - | GenericQueryCtx; - /** * Define a query in this Convex app's public API. * @@ -36,7 +38,7 @@ type GenericCtx = * @param func - The query function. It receives a {@link QueryCtx} as its first argument. * @returns The wrapped query. Include this as an `export` to name it and make it accessible. */ -export declare const query: QueryBuilder; +export const query: QueryBuilder = queryGeneric; /** * Define a query that is only accessible from other Convex functions (but not from the client). @@ -46,7 +48,8 @@ export declare const query: QueryBuilder; * @param func - The query function. It receives a {@link QueryCtx} as its first argument. * @returns The wrapped query. Include this as an `export` to name it and make it accessible. */ -export declare const internalQuery: QueryBuilder; +export const internalQuery: QueryBuilder = + internalQueryGeneric; /** * Define a mutation in this Convex app's public API. @@ -56,7 +59,7 @@ export declare const internalQuery: QueryBuilder; * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. */ -export declare const mutation: MutationBuilder; +export const mutation: MutationBuilder = mutationGeneric; /** * Define a mutation that is only accessible from other Convex functions (but not from the client). @@ -66,7 +69,8 @@ export declare const mutation: MutationBuilder; * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. */ -export declare const internalMutation: MutationBuilder; +export const internalMutation: MutationBuilder = + internalMutationGeneric; /** * Define an action in this Convex app's public API. @@ -79,7 +83,7 @@ export declare const internalMutation: MutationBuilder; * @param func - The action. It receives an {@link ActionCtx} as its first argument. * @returns The wrapped action. Include this as an `export` to name it and make it accessible. */ -export declare const action: ActionBuilder; +export const action: ActionBuilder = actionGeneric; /** * Define an action that is only accessible from other Convex functions (but not from the client). @@ -87,19 +91,26 @@ export declare const action: ActionBuilder; * @param func - The function. It receives an {@link ActionCtx} as its first argument. * @returns The wrapped function. Include this as an `export` to name it and make it accessible. */ -export declare const internalAction: ActionBuilder; +export const internalAction: ActionBuilder = + internalActionGeneric; /** * Define an HTTP action. * - * This function will be used to respond to HTTP requests received by a Convex - * deployment if the requests matches the path and method where this action - * is routed. Be sure to route your action in `convex/http.js`. + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. * - * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. */ -export declare const httpAction: HttpActionBuilder; +export const httpAction: HttpActionBuilder = httpActionGeneric; + +type GenericCtx = + | GenericActionCtx + | GenericMutationCtx + | GenericQueryCtx; /** * A set of services for use within Convex query functions. @@ -107,8 +118,7 @@ export declare const httpAction: HttpActionBuilder; * The query context is passed as the first argument to any Convex query * function run on the server. * - * This differs from the {@link MutationCtx} because all of the services are - * read-only. + * If you're using code generation, use the `QueryCtx` type in `convex/_generated/server.d.ts` instead. */ export type QueryCtx = GenericQueryCtx; @@ -117,6 +127,8 @@ export type QueryCtx = GenericQueryCtx; * * The mutation context is passed as the first argument to any Convex mutation * function run on the server. + * + * If you're using code generation, use the `MutationCtx` type in `convex/_generated/server.d.ts` instead. */ export type MutationCtx = GenericMutationCtx; diff --git a/src/component/chunks.test.ts b/src/component/chunks.test.ts index ac32fca..c15d32e 100644 --- a/src/component/chunks.test.ts +++ b/src/component/chunks.test.ts @@ -30,7 +30,7 @@ describe("chunks", () => { namespaceId: Id<"namespaces">, key = "test-entry", version = 0, - status: "ready" | "pending" = "ready" + status: "ready" | "pending" = "ready", ) { return await t.run(async (ctx) => { return ctx.db.insert("entries", { @@ -70,7 +70,7 @@ describe("chunks", () => { startOrder: 0, chunks, }); - }) + }), ).rejects.toThrow(`Entry ${nonExistentDocId} not found`); }); @@ -141,10 +141,10 @@ describe("chunks", () => { expect(overwrittenChunk2).toBeDefined(); const content1 = await t.run(async (ctx) => - ctx.db.get(overwrittenChunk1!.contentId) + ctx.db.get(overwrittenChunk1!.contentId), ); const content2 = await t.run(async (ctx) => - ctx.db.get(overwrittenChunk2!.contentId) + ctx.db.get(overwrittenChunk2!.contentId), ); expect(content1!.text).toBe("Overwritten chunk 1 content"); @@ -176,7 +176,7 @@ describe("chunks", () => { namespaceId, "versioned-entry", 2, - "pending" + "pending", ); // Insert chunks in version 2 (this should mark v1 chunks as replaced) @@ -279,10 +279,10 @@ describe("chunks", () => { // Verify chunk content const doc1Content0 = await t.run(async (ctx) => - ctx.db.get(doc1ChunksList[0].contentId) + ctx.db.get(doc1ChunksList[0].contentId), ); const doc2Content0 = await t.run(async (ctx) => - ctx.db.get(doc2ChunksList[0].contentId) + ctx.db.get(doc2ChunksList[0].contentId), ); expect(doc1Content0!.text).toBe("Test chunk content 1"); @@ -309,7 +309,7 @@ describe("chunks", () => { return ctx.db .query("chunks") .withIndex("entryId_order", (q) => - q.eq("entryId", entryId).eq("order", 2) + q.eq("entryId", entryId).eq("order", 2), ) .first(); }); @@ -319,7 +319,7 @@ describe("chunks", () => { // Verify content const content = await t.run(async (ctx) => - ctx.db.get(singleChunk!.contentId) + ctx.db.get(singleChunk!.contentId), ); expect(content!.text).toBe("Test chunk content 3"); }); @@ -467,7 +467,7 @@ describe("chunks", () => { { embeddingIds: [chunkDocs[2].state.embeddingId], chunkContext: { before: 1, after: 2 }, - } + }, ); expect(entries).toHaveLength(1); expect(entries[0].entryId).toBe(entryId); @@ -532,7 +532,7 @@ describe("chunks", () => { doc2ChunkDocs[2].state.embeddingId, // doc2, chunk at order 2 ], chunkContext: { before: 1, after: 1 }, - } + }, ); expect(entries).toHaveLength(2); @@ -562,7 +562,7 @@ describe("chunks", () => { namespaceId, "versioned-entry", 1, - "ready" + "ready", ); // Insert chunks in version 1 @@ -580,7 +580,7 @@ describe("chunks", () => { namespaceId, "versioned-entry", 2, - "pending" + "pending", ); // Insert chunks in version 2 @@ -631,7 +631,7 @@ describe("chunks", () => { v2ChunkDocs[1].state.embeddingId, // v2, chunk at order 1 ], chunkContext: { before: 1, after: 1 }, - } + }, ); expect(entries).toHaveLength(2); diff --git a/src/component/chunks.ts b/src/component/chunks.ts index 8ffbe5c..6527b5b 100644 --- a/src/component/chunks.ts +++ b/src/component/chunks.ts @@ -50,7 +50,7 @@ export const insert = mutation({ export async function insertChunks( ctx: MutationCtx, - { entryId, startOrder, chunks }: InsertChunksArgs + { entryId, startOrder, chunks }: InsertChunksArgs, ) { const entry = await ctx.db.get(entryId); if (!entry) { @@ -71,12 +71,12 @@ export async function insertChunks( q .eq("entryId", entryId) .gte("order", startOrder) - .lt("order", startOrder + chunks.length) + .lt("order", startOrder + chunks.length), ) .collect(); if (existingChunks.length > 0) { console.debug( - `Deleting ${existingChunks.length} existing chunks for entry ${entryId} at version ${entry.version}` + `Deleting ${existingChunks.length} existing chunks for entry ${entryId} at version ${entry.version}`, ); } // TODO: avoid writing if they're the same @@ -87,11 +87,11 @@ export async function insertChunks( } await ctx.db.delete(c.contentId); await ctx.db.delete(c._id); - }) + }), ); const numberedFilter = numberedFilterFromNamedFilters( entry.filterValues, - namespace!.filterNames + namespace!.filterNames, ); for (const chunk of chunks) { const contentId = await ctx.db.insert("content", { @@ -110,7 +110,7 @@ export async function insertChunks( chunk.embedding, entry.namespaceId, entry.importance, - numberedFilter + numberedFilter, ); state = { kind: "ready", @@ -126,7 +126,7 @@ export async function insertChunks( contentId, namespaceId: entry.namespaceId, ...filterFieldsFromNumbers(entry.namespaceId, numberedFilter), - }) + }), ); order++; } @@ -146,14 +146,14 @@ async function ensureLatestEntryVersion(ctx: QueryCtx, entry: Doc<"entries">) { .eq("namespaceId", entry.namespaceId) .eq("status.kind", status) .eq("key", entry.key) - .gt("version", entry.version) - ) + .gt("version", entry.version), + ), ), - ["version"] + ["version"], ).first(); if (newerEntry) { console.warn( - `Bailing from inserting chunks for entry ${entry.key} at version ${entry.version} since there's a newer version ${newerEntry.version} (status ${newerEntry.status}) creation time difference ${(newerEntry._creationTime - entry._creationTime).toFixed(0)}ms` + `Bailing from inserting chunks for entry ${entry.key} at version ${entry.version} since there's a newer version ${newerEntry.version} (status ${newerEntry.status}) creation time difference ${(newerEntry._creationTime - entry._creationTime).toFixed(0)}ms`, ); return false; } @@ -189,7 +189,7 @@ export const replaceChunksPage = mutation({ q .eq("namespaceId", entry.namespaceId) .eq("status.kind", "pending") - .eq("key", entry.key) + .eq("key", entry.key), ) .collect() ).filter((e) => e._id !== entry._id) @@ -201,25 +201,25 @@ export const replaceChunksPage = mutation({ stream(ctx.db, schema) .query("chunks") .withIndex("entryId_order", (q) => - q.eq("entryId", entry._id).gte("order", startOrder) - ) + q.eq("entryId", entry._id).gte("order", startOrder), + ), ), - ["order"] + ["order"], ); const namespaceId = entry.namespaceId; const namedFilters = numberedFilterFromNamedFilters( entry.filterValues, - namespace!.filterNames + namespace!.filterNames, ); async function addChunk( - chunk: Doc<"chunks"> & { state: { kind: "pending" } } + chunk: Doc<"chunks"> & { state: { kind: "pending" } }, ) { const embeddingId = await insertEmbedding( ctx, chunk.state.embedding, namespaceId, entry.importance, - namedFilters + namedFilters, ); await ctx.db.patch(chunk._id, { state: { kind: "ready", embeddingId } }); } @@ -243,7 +243,7 @@ export const replaceChunksPage = mutation({ pendingSearchableText: chunk.state.searchableText, }, }); - }) + }), ); chunksToDeleteEmbeddings = []; if (chunkToAdd) { @@ -274,7 +274,7 @@ export const replaceChunksPage = mutation({ if (chunk.entryId === entryId) { if (chunkToAdd) { console.warn( - `Multiple pending chunks before changing order ${chunk.order} for entry ${entryId} version ${entry.version}: ${chunkToAdd._id} and ${chunk._id}` + `Multiple pending chunks before changing order ${chunk.order} for entry ${entryId} version ${entry.version}: ${chunkToAdd._id} and ${chunk._id}`, ); await addChunk(chunkToAdd); } @@ -285,7 +285,7 @@ export const replaceChunksPage = mutation({ chunksToDeleteEmbeddings.push(chunk); } else { console.debug( - `Skipping adding chunk ${chunk._id} for entry ${entryId} version ${entry.version} since it's already ready` + `Skipping adding chunk ${chunk._id} for entry ${entryId} version ${entry.version} since it's already ready`, ); } } @@ -305,7 +305,7 @@ export const vRangeResult = v.object({ v.object({ text: v.string(), metadata: v.optional(v.record(v.string(), v.any())), - }) + }), ), }); @@ -320,7 +320,7 @@ export const getRangesOfChunks = internalQuery({ }), handler: async ( ctx, - args + args, ): Promise<{ ranges: (null | Infer)[]; entries: Entry[]; @@ -331,19 +331,19 @@ export const getRangesOfChunks = internalQuery({ ctx.db .query("chunks") .withIndex("embeddingId", (q) => - q.eq("state.embeddingId", embeddingId) + q.eq("state.embeddingId", embeddingId), ) .order("desc") - .first() - ) + .first(), + ), ); // Note: This preserves order of entries as they first appeared. const entries = ( await Promise.all( Array.from( - new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)) - ).map((id) => ctx.db.get(id)) + new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)), + ).map((id) => ctx.db.get(id)), ) ) .filter((d) => d !== null) @@ -361,7 +361,7 @@ export const getRangesOfChunks = internalQuery({ acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b); return acc; }, - {} as Record, number[]> + {} as Record, number[]>, ); const result: Array | null> = []; @@ -375,7 +375,7 @@ export const getRangesOfChunks = internalQuery({ // instead we'd check that other chunks are not the same doc/order if ( result.find( - (r) => r?.entryId === chunk.entryId && r?.order === chunk.order + (r) => r?.entryId === chunk.entryId && r?.order === chunk.order, ) ) { // De-dupe chunks @@ -393,12 +393,12 @@ export const getRangesOfChunks = internalQuery({ const startOrder = Math.max( chunk.order - chunkContext.before, 0, - Math.min(previousOrder + 1, chunk.order) + Math.min(previousOrder + 1, chunk.order), ); // We stop short if the next chunk order's "before" context will cover it. const endOrder = Math.min( chunk.order + chunkContext.after + 1, - Math.max(nextOrder - chunkContext.before, chunk.order + 1) + Math.max(nextOrder - chunkContext.before, chunk.order + 1), ); const contentIds: Id<"content">[] = []; if (startOrder === chunk.order && endOrder === chunk.order + 1) { @@ -410,7 +410,7 @@ export const getRangesOfChunks = internalQuery({ q .eq("entryId", entryId) .gte("order", startOrder) - .lt("order", endOrder) + .lt("order", endOrder), ) .collect(); for (const chunk of chunks) { @@ -422,7 +422,7 @@ export const getRangesOfChunks = internalQuery({ const content = await ctx.db.get(contentId); assert(content, `Content ${contentId} not found`); return { text: content.text, metadata: content.metadata }; - }) + }), ); result.push({ entryId, order: chunk.order, startOrder, content }); @@ -453,7 +453,7 @@ export const list = query({ const content = await ctx.db.get(chunk.contentId); assert(content, `Content ${chunk.contentId} not found`); return publicChunk(chunk, content); - }) + }), ), }; }, @@ -493,12 +493,12 @@ export const deleteChunksPage = internalMutation({ export async function deleteChunksPageHandler( ctx: MutationCtx, - { entryId, startOrder }: { entryId: Id<"entries">; startOrder: number } + { entryId, startOrder }: { entryId: Id<"entries">; startOrder: number }, ) { const chunkStream = ctx.db .query("chunks") .withIndex("entryId_order", (q) => - q.eq("entryId", entryId).gte("order", startOrder) + q.eq("entryId", entryId).gte("order", startOrder), ); let dataUsedSoFar = 0; for await (const chunk of chunkStream) { @@ -557,7 +557,7 @@ async function estimateContentSize(ctx: QueryCtx, contentId: Id<"content">) { if (content) { dataUsedSoFar += content.text.length; dataUsedSoFar += JSON.stringify( - convexToJson(content.metadata ?? {}) + convexToJson(content.metadata ?? {}), ).length; } return dataUsedSoFar; diff --git a/src/component/embeddings/importance.test.ts b/src/component/embeddings/importance.test.ts index 0d42169..b43704f 100644 --- a/src/component/embeddings/importance.test.ts +++ b/src/component/embeddings/importance.test.ts @@ -45,7 +45,7 @@ describe("importance.ts", () => { expect(result[0]).toBeCloseTo(embedding[0] * importance); expect(result[1]).toBeCloseTo(embedding[1] * importance); expect( - Math.sqrt(result[0] ** 2 + result[1] ** 2 + result[2] ** 2) + Math.sqrt(result[0] ** 2 + result[1] ** 2 + result[2] ** 2), ).toBeCloseTo(1); }); @@ -175,7 +175,7 @@ describe("importance.ts", () => { expect(retrievedImportance).toBeCloseTo(importance, 3); expect(Math.abs(retrievedImportance - importance)).toBeLessThan( - tolerance + tolerance, ); }); }); @@ -198,13 +198,13 @@ describe("importance.ts", () => { // Create vector with initial importance const vectorWithInitialImp = vectorWithImportance( embedding, - initialImportance + initialImportance, ); // Modify importance const vectorWithModifiedImp = modifyImportance( vectorWithInitialImp, - newImportance + newImportance, ); // Retrieve and verify diff --git a/src/component/embeddings/importance.ts b/src/component/embeddings/importance.ts index c1637d9..fa4cd3a 100644 --- a/src/component/embeddings/importance.ts +++ b/src/component/embeddings/importance.ts @@ -35,7 +35,7 @@ export function vectorWithImportance(embedding: number[], importance: number) { // We drop the final dimension if it'd make it larger than 4096. // Unfortunate current limitation of Convex vector search. const vectorToModify = normalizeVector( - embedding.length === 4096 ? embedding.slice(0, 4095) : embedding + embedding.length === 4096 ? embedding.slice(0, 4095) : embedding, ); const scaled = scaleVector(vectorToModify, importance); diff --git a/src/component/embeddings/index.test.ts b/src/component/embeddings/index.test.ts index 3d238d6..7048b78 100644 --- a/src/component/embeddings/index.test.ts +++ b/src/component/embeddings/index.test.ts @@ -25,7 +25,6 @@ const testApi: ApiFromModules<{ fns: { search: typeof search; }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any }>["fns"] = anyApi["embeddings"]["index.test"] as any; describe("embeddings", () => { @@ -62,7 +61,7 @@ describe("embeddings", () => { expect(insertedVector).toBeDefined(); expect(insertedVector!.namespaceId).toBe(namespaceId); expect(insertedVector!.vector).toHaveLength( - vectorWithImportanceDimension(128) + vectorWithImportanceDimension(128), ); expect(insertedVector!.filter0).toBeUndefined(); expect(insertedVector!.filter1).toBeUndefined(); @@ -94,7 +93,7 @@ describe("embeddings", () => { embedding, namespaceId, importance, - undefined + undefined, ); }); @@ -116,7 +115,7 @@ describe("embeddings", () => { // Vectors should be different due to importance scaling expect(insertedVector!.vector).not.toEqual( - vectorWithoutImportanceData!.vector + vectorWithoutImportanceData!.vector, ); // The last element should be the weight: sqrt(1 - importance^2) diff --git a/src/component/embeddings/index.ts b/src/component/embeddings/index.ts index 9522944..7c10a1b 100644 --- a/src/component/embeddings/index.ts +++ b/src/component/embeddings/index.ts @@ -47,7 +47,7 @@ export async function insertEmbedding( embedding: number[], namespaceId: Id<"namespaces">, importance: number | undefined, - filters: NumberedFilter | undefined + filters: NumberedFilter | undefined, ) { const filterFields = filterFieldsFromNumbers(namespaceId, filters); const dimension = validateVectorDimension(embedding.length); @@ -74,12 +74,12 @@ export async function searchEmbeddings( // filter3, filter1, or filter2 is present. filters: Array; limit: number; - } + }, ) { const dimension = validateVectorDimension(embedding.length); const tableName = getVectorTableName(dimension); const orFilters = filters.flatMap((filter) => - filterFieldsFromNumbers(namespaceId, filter) + filterFieldsFromNumbers(namespaceId, filter), ); return ctx.vectorSearch(tableName, "vector", { vector: searchVector(embedding), @@ -89,9 +89,9 @@ export async function searchEmbeddings( : q.or( ...orFilters.flatMap((namedFilter) => Object.entries(namedFilter).map(([filterField, filter]) => - q.eq(filterField as keyof (typeof orFilters)[number], filter) - ) - ) + q.eq(filterField as keyof (typeof orFilters)[number], filter), + ), + ), ), limit, }); diff --git a/src/component/embeddings/tables.ts b/src/component/embeddings/tables.ts index a09ccc8..6692294 100644 --- a/src/component/embeddings/tables.ts +++ b/src/component/embeddings/tables.ts @@ -56,11 +56,11 @@ export const VectorDimensions = [ ] as const; export function assertVectorDimension( - dimension: number + dimension: number, ): asserts dimension is VectorDimension { if (!VectorDimensions.includes(dimension as VectorDimension)) { throw new Error( - `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}` + `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`, ); } } @@ -68,14 +68,14 @@ export function assertVectorDimension( export function validateVectorDimension(dimension: number): VectorDimension { if (!VectorDimensions.includes(dimension as VectorDimension)) { throw new Error( - `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}` + `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`, ); } return dimension as VectorDimension; } export type VectorDimension = (typeof VectorDimensions)[number]; export const VectorTableNames = VectorDimensions.map( - (d) => `vectors_${d}` + (d) => `vectors_${d}`, ) as `vectors_${(typeof VectorDimensions)[number]}`[]; export type VectorTableName = (typeof VectorTableNames)[number]; export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>; @@ -83,7 +83,7 @@ export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>; export const vVectorDimension = literals(...VectorDimensions); export const vVectorTableName = literals(...VectorTableNames); export const vVectorId = v.union( - ...VectorTableNames.map((name) => v.id(name)) + ...VectorTableNames.map((name) => v.id(name)), ) as VUnion< GenericId<(typeof VectorTableNames)[number]>, VId<(typeof VectorTableNames)[number]>[] @@ -109,7 +109,7 @@ const tables: { VectorDimensions.map((dimensions) => [ `vectors_${dimensions}`, table(dimensions), - ]) + ]), ) as Record<`vectors_${(typeof VectorDimensions)[number]}`, Table>; export default tables; diff --git a/src/component/entries.test.ts b/src/component/entries.test.ts index 3c02303..0282fe6 100644 --- a/src/component/entries.test.ts +++ b/src/component/entries.test.ts @@ -97,8 +97,8 @@ describe("entries", () => { .filter((q) => q.and( q.eq(q.field("namespaceId"), namespaceId), - q.eq(q.field("key"), entry.key) - ) + q.eq(q.field("key"), entry.key), + ), ) .collect(); }); @@ -143,8 +143,8 @@ describe("entries", () => { .filter((q) => q.and( q.eq(q.field("namespaceId"), namespaceId), - q.eq(q.field("key"), entry.key) - ) + q.eq(q.field("key"), entry.key), + ), ) .collect(); }); @@ -537,7 +537,7 @@ describe("entries", () => { { namespaceId, key: "shared-key", - } + }, ); expect(sharedBefore).toHaveLength(2); @@ -563,7 +563,7 @@ describe("entries", () => { const sharedAfter = await t.query( internal.entries.getEntriesForNamespaceByKey, - { namespaceId, key: "shared-key" } + { namespaceId, key: "shared-key" }, ); expect(sharedAfter).toHaveLength(0); @@ -722,8 +722,8 @@ describe("entries", () => { .filter((q) => q.and( q.eq(q.field("namespaceId"), namespaceId), - q.eq(q.field("key"), "versioned-key") - ) + q.eq(q.field("key"), "versioned-key"), + ), ) .collect(); }); @@ -747,8 +747,8 @@ describe("entries", () => { .filter((q) => q.and( q.eq(q.field("namespaceId"), namespaceId), - q.eq(q.field("key"), "versioned-key") - ) + q.eq(q.field("key"), "versioned-key"), + ), ) .collect(); }); diff --git a/src/component/entries.ts b/src/component/entries.ts index 1defbc1..0b75705 100644 --- a/src/component/entries.ts +++ b/src/component/entries.ts @@ -111,7 +111,7 @@ export const addAsync = mutation({ name: workpoolName(namespace.namespace, args.entry.key, entryId), onComplete: internal.entries.addAsyncOnComplete, context: entryId, - } + }, ); return { entryId, status: status.kind, created: true }; }, @@ -120,7 +120,7 @@ export const addAsync = mutation({ function workpoolName( namespace: string, key: string | undefined, - entryId: Id<"entries"> + entryId: Id<"entries">, ) { return `rag-async-${namespace}-${key ? key + "-" + entryId : entryId}`; } @@ -137,7 +137,7 @@ export const addAsyncOnComplete = internalMutation({ const entry = await ctx.db.get(args.context); if (!entry) { console.error( - `Entry ${args.context} not found when trying to complete chunker for async add` + `Entry ${args.context} not found when trying to complete chunker for async add`, ); return; } @@ -154,7 +154,7 @@ export const addAsyncOnComplete = internalMutation({ namespace, entry, null, - args.result.kind === "canceled" ? "Canceled" : args.result.error + args.result.kind === "canceled" ? "Canceled" : args.result.error, ); } } @@ -169,7 +169,7 @@ type AddEntryArgs = Pick< async function findExistingEntry( ctx: MutationCtx, namespaceId: Id<"namespaces">, - key: string | undefined + key: string | undefined, ) { if (!key) { return null; @@ -182,11 +182,11 @@ async function findExistingEntry( q .eq("namespaceId", namespaceId) .eq("status.kind", status) - .eq("key", key) + .eq("key", key), ) - .order("desc") + .order("desc"), ), - ["version"] + ["version"], ).first(); return existing; } @@ -256,7 +256,7 @@ async function runOnComplete( namespace: Doc<"namespaces">, entry: Doc<"entries">, replacedEntry: Doc<"entries"> | null, - error?: string + error?: string, ) { await ctx.runMutation(onComplete as unknown as OnComplete, { namespace: publicNamespace(namespace), @@ -282,8 +282,8 @@ function entryIsSame(existing: Doc<"entries">, newEntry: AddEntryArgs) { if ( !existing.filterValues.every((filter) => newEntry.filterValues.some( - (f) => f.name === filter.name && f.value === filter.value - ) + (f) => f.name === filter.name && f.value === filter.value, + ), ) ) { return false; @@ -309,7 +309,7 @@ export const list = query({ .withIndex("status_namespaceId", (q) => namespaceId ? q.eq("status.kind", args.status).eq("namespaceId", namespaceId) - : q.eq("status.kind", args.status) + : q.eq("status.kind", args.status), ) .order(args.order ?? "asc") .paginate(args.paginationOpts); @@ -359,16 +359,16 @@ export const findByContentHash = query({ q .eq("namespaceId", namespace._id) .eq("status.kind", status) - .eq("key", args.key) + .eq("key", args.key), ) - .order("desc") + .order("desc"), ), - ["version"] + ["version"], )) { attempts++; if (attempts > 20) { console.debug( - `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null` + `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null`, ); return null; } @@ -407,7 +407,7 @@ export const promoteToReady = mutation({ async function promoteToReadyHandler( ctx: MutationCtx, - args: { entryId: Id<"entries"> } + args: { entryId: Id<"entries"> }, ) { const entry = await ctx.db.get(args.entryId); assert(entry, `Entry ${args.entryId} not found`); @@ -418,7 +418,7 @@ async function promoteToReadyHandler( return { replacedEntry: null }; } else if (entry.status.kind === "replaced") { console.debug( - `Entry ${args.entryId} is already replaced, returning the current version...` + `Entry ${args.entryId} is already replaced, returning the current version...`, ); return { replacedEntry: publicEntry(entry) }; } @@ -441,7 +441,7 @@ async function promoteToReadyHandler( previousStatus.onComplete, namespace, entry, - previousEntry + previousEntry, ); } // Then mark all previous pending entries as replaced, @@ -454,7 +454,7 @@ async function promoteToReadyHandler( .eq("namespaceId", entry.namespaceId) .eq("status.kind", "pending") .eq("key", entry.key) - .lt("version", entry.version) + .lt("version", entry.version), ) .collect(); await Promise.all( @@ -468,10 +468,10 @@ async function promoteToReadyHandler( previousStatus.onComplete, namespace, entry, - null + null, ); } - }) + }), ); } return { @@ -489,7 +489,7 @@ export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) { q .eq("namespaceId", entry.namespaceId) .eq("status.kind", "ready") - .eq("key", entry.key) + .eq("key", entry.key), ) .unique(); if (previousEntry?._id === entry._id) return null; @@ -542,7 +542,7 @@ export const deleteAsync = mutation({ async function deleteAsyncHandler( ctx: MutationCtx, - args: { entryId: Id<"entries">; startOrder: number } + args: { entryId: Id<"entries">; startOrder: number }, ) { const { entryId, startOrder } = args; const entry = await ctx.db.get(entryId); @@ -614,7 +614,7 @@ export const deleteByKeyAsync = mutation({ async function getEntriesByKey( ctx: QueryCtx, - args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number } + args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number }, ): Promise[]> { return mergedStream( statuses.map((status) => @@ -625,11 +625,11 @@ async function getEntriesByKey( .eq("namespaceId", args.namespaceId) .eq("status.kind", status) .eq("key", args.key) - .lt("version", args.beforeVersion ?? Infinity) + .lt("version", args.beforeVersion ?? Infinity), ) - .order("desc") + .order("desc"), ), - ["version"] + ["version"], ).take(100); } @@ -653,7 +653,7 @@ export const deleteByKeySync = action({ while (true) { const entries: Doc<"entries">[] = await ctx.runQuery( internal.entries.getEntriesForNamespaceByKey, - { namespaceId: args.namespaceId, key: args.key } + { namespaceId: args.namespaceId, key: args.key }, ); for await (const entry of entries) { await ctx.runAction(api.entries.deleteSync, { diff --git a/src/component/filters.ts b/src/component/filters.ts index cf589cb..bea029a 100644 --- a/src/component/filters.ts +++ b/src/component/filters.ts @@ -55,7 +55,7 @@ export type NamedFilter = { */ export function filterFieldsFromNumbers( namespaceId: GenericId<"namespaces">, - filters: NumberedFilter | undefined + filters: NumberedFilter | undefined, ): NamedFilterField { const filterFields: NamedFilterField = {}; if (!filters) return filterFields; @@ -63,7 +63,7 @@ export function filterFieldsFromNumbers( const index = Number(i); if (isNaN(index) || index < 0 || index >= filterFieldNames.length) { console.warn( - `Unknown filter index: ${index} for value ${JSON.stringify(filter)}` + `Unknown filter index: ${index} for value ${JSON.stringify(filter)}`, ); break; } @@ -78,7 +78,7 @@ export function filterFieldsFromNumbers( */ export function numberedFilterFromNamedFilters( namedFilters: Array<{ name: string; value: Value }>, - filterNames: string[] + filterNames: string[], ): NumberedFilter { const numberedFilter: NumberedFilter = {}; for (const namedFilter of namedFilters) { @@ -86,8 +86,8 @@ export function numberedFilterFromNamedFilters( if (index === -1) { throw new Error( `Unknown filter name: ${namedFilter.name} for namespace with names ${filterNames.join( - ", " - )}` + ", ", + )}`, ); } numberedFilter[index] = namedFilter.value; @@ -101,7 +101,7 @@ export function numberedFilterFromNamedFilters( */ export function numberedFiltersFromNamedFilters( filters: NamedFilter[], - filterNames: string[] + filterNames: string[], ): Array { const filterFields: Array = []; for (const filter of filters) { @@ -109,8 +109,8 @@ export function numberedFiltersFromNamedFilters( if (index === -1) { throw new Error( `Unknown filter name: ${filter.name} for namespace with names ${filterNames.join( - ", " - )}` + ", ", + )}`, ); } filterFields.push({ [index]: filter.value }); diff --git a/src/component/namespaces.ts b/src/component/namespaces.ts index 0539782..74b2644 100644 --- a/src/component/namespaces.ts +++ b/src/component/namespaces.ts @@ -33,7 +33,7 @@ function namespaceIsCompatible( modelId: string; dimension: number; filterNames: string[]; - } + }, ) { // Check basic compatibility if ( @@ -79,12 +79,12 @@ export const getCompatibleNamespace = internalQuery({ export async function getCompatibleNamespaceHandler( ctx: QueryCtx, - args: ObjectType + args: ObjectType, ) { const iter = ctx.db .query("namespaces") .withIndex("status_namespace_version", (q) => - q.eq("status.kind", "ready").eq("namespace", args.namespace) + q.eq("status.kind", "ready").eq("namespace", args.namespace), ) .order("desc"); for await (const existing of iter) { @@ -132,11 +132,11 @@ export const getOrCreate = mutation({ stream(ctx.db, schema) .query("namespaces") .withIndex("status_namespace_version", (q) => - q.eq("status.kind", status).eq("namespace", args.namespace) + q.eq("status.kind", status).eq("namespace", args.namespace), ) - .order("desc") + .order("desc"), ), - ["version"] + ["version"], ); let version: number = 0; @@ -172,7 +172,7 @@ async function runOnComplete( ctx: MutationCtx, onComplete: string | undefined, namespace: Doc<"namespaces">, - replacedNamespace: Doc<"namespaces"> | null + replacedNamespace: Doc<"namespaces"> | null, ) { const onCompleteFn = onComplete as unknown as OnCompleteNamespace; if (!onCompleteFn) { @@ -198,25 +198,25 @@ export const promoteToReady = mutation({ async function promoteToReadyHandler( ctx: MutationCtx, - args: { namespaceId: Id<"namespaces"> } + args: { namespaceId: Id<"namespaces"> }, ) { const namespace = await ctx.db.get(args.namespaceId); assert(namespace, `Namespace ${args.namespaceId} not found`); if (namespace.status.kind === "ready") { console.debug( - `Namespace ${args.namespaceId} is already ready, not promoting` + `Namespace ${args.namespaceId} is already ready, not promoting`, ); return { replacedNamespace: null }; } else if (namespace.status.kind === "replaced") { console.debug( - `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself` + `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself`, ); return { replacedNamespace: publicNamespace(namespace) }; } const previousNamespace = await ctx.db .query("namespaces") .withIndex("status_namespace_version", (q) => - q.eq("status.kind", "ready").eq("namespace", namespace.namespace) + q.eq("status.kind", "ready").eq("namespace", namespace.namespace), ) .order("desc") .unique(); @@ -237,7 +237,7 @@ async function promoteToReadyHandler( ctx, previousStatus.onComplete, namespace, - previousNamespace + previousNamespace, ); } const previousPendingNamespaces = await ctx.db @@ -246,7 +246,7 @@ async function promoteToReadyHandler( q .eq("status.kind", "pending") .eq("namespace", namespace.namespace) - .lt("version", namespace.version) + .lt("version", namespace.version), ) .collect(); // Then mark all previous pending namespaces as replaced, @@ -259,7 +259,7 @@ async function promoteToReadyHandler( if (previousStatus.kind === "pending" && previousStatus.onComplete) { await runOnComplete(ctx, previousStatus.onComplete, namespace, null); } - }) + }), ); return { replacedNamespace: previousNamespace @@ -278,7 +278,7 @@ export const list = query({ const namespaces = await paginator(ctx.db, schema) .query("namespaces") .withIndex("status_namespace_version", (q) => - q.eq("status.kind", args.status ?? "ready") + q.eq("status.kind", args.status ?? "ready"), ) .order("desc") .paginate(args.paginationOpts); @@ -298,11 +298,11 @@ export const listNamespaceVersions = query({ stream(ctx.db, schema) .query("namespaces") .withIndex("status_namespace_version", (q) => - q.eq("status.kind", status).eq("namespace", args.namespace) + q.eq("status.kind", status).eq("namespace", args.namespace), ) - .order("desc") + .order("desc"), ), - ["version"] + ["version"], ).paginate(args.paginationOpts); return { @@ -332,21 +332,21 @@ export const deleteNamespace = mutation({ async function deleteHandler( ctx: MutationCtx, - args: { namespaceId: Id<"namespaces"> } + args: { namespaceId: Id<"namespaces"> }, ) { const namespace = await ctx.db.get(args.namespaceId); assert(namespace, `Namespace ${args.namespaceId} not found`); const anyEntry = await ctx.db .query("entries") .withIndex("namespaceId_status_key_version", (q) => - q.eq("namespaceId", args.namespaceId) + q.eq("namespaceId", args.namespaceId), ) .first(); if (anyEntry) { throw new Error( `Namespace ${args.namespaceId} cannot delete, has entries` + "First delete all entries." + - `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})` + `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})`, ); } await ctx.db.delete(args.namespaceId); diff --git a/src/component/schema.ts b/src/component/schema.ts index a500fd6..2da8d6d 100644 --- a/src/component/schema.ts +++ b/src/component/schema.ts @@ -20,7 +20,7 @@ export const vStatusWithOnComplete = v.union( v.object({ kind: v.literal("replaced"), replacedAt: v.number(), - }) + }), ); export type StatusWithOnComplete = Infer; @@ -79,7 +79,7 @@ export const schema = defineSchema({ embeddingId: vVectorId, vector: v.array(v.number()), pendingSearchableText: v.optional(v.string()), - }) + }), ), // TODO: should content be inline? contentId: v.id("content"), diff --git a/src/component/search.test.ts b/src/component/search.test.ts index 6859f01..55c4aac 100644 --- a/src/component/search.test.ts +++ b/src/component/search.test.ts @@ -16,7 +16,7 @@ describe("search", () => { t: ConvexTest, namespace = "test-namespace", dimension = 128, - filterNames: string[] = [] + filterNames: string[] = [], ) { return await t.run(async (ctx) => { return ctx.db.insert("namespaces", { @@ -35,7 +35,7 @@ describe("search", () => { namespaceId: Id<"namespaces">, key = "test-entry", version = 0, - filterValues: Array<{ name: string; value: Value }> = [] + filterValues: Array<{ name: string; value: Value }> = [], ) { return await t.run(async (ctx) => { return ctx.db.insert("entries", { @@ -207,7 +207,7 @@ describe("search", () => { // With threshold should return fewer results expect(resultWithThreshold.results.length).toBeLessThan( - resultWithoutThreshold.results.length + resultWithoutThreshold.results.length, ); expect(resultWithoutThreshold.results).toHaveLength(2); @@ -305,7 +305,7 @@ describe("search", () => { t, "multi-filter-namespace", 128, - ["category", "priority_category"] + ["category", "priority_category"], ); // Create entries with different filter combinations @@ -438,7 +438,7 @@ describe("search", () => { // Results should be sorted by score (best first) for (let i = 1; i < result.results.length; i++) { expect(result.results[i - 1].score).toBeGreaterThanOrEqual( - result.results[i].score + result.results[i].score, ); } }); diff --git a/src/component/search.ts b/src/component/search.ts index f46a27d..00e2652 100644 --- a/src/component/search.ts +++ b/src/component/search.ts @@ -21,14 +21,20 @@ export const search = action({ limit: v.number(), vectorScoreThreshold: v.optional(v.number()), chunkContext: v.optional( - v.object({ before: v.number(), after: v.number() }) + v.object({ before: v.number(), after: v.number() }), ), }, returns: v.object({ results: v.array(vSearchResult), entries: v.array(vEntry), }), - handler: async (ctx, args) => { + handler: async ( + ctx, + args, + ): Promise<{ + results: SearchResult[]; + entries: Infer[]; + }> => { const { modelId, embedding, filters, limit } = args; const namespace = await ctx.runQuery( internal.namespaces.getCompatibleNamespace, @@ -37,11 +43,11 @@ export const search = action({ modelId, dimension: embedding.length, filterNames: filters.map((f) => f.name), - } + }, ); if (!namespace) { console.debug( - `No compatible namespace found for ${args.namespace} with model ${args.modelId} and dimension ${embedding.length} and filters ${filters.map((f) => f.name).join(", ")}.` + `No compatible namespace found for ${args.namespace} with model ${args.modelId} and dimension ${embedding.length} and filters ${filters.map((f) => f.name).join(", ")}.`, ); return { results: [], @@ -64,7 +70,7 @@ export const search = action({ { embeddingIds: aboveThreshold.map((r) => r._id), chunkContext, - } + }, ); return { results: ranges @@ -77,7 +83,7 @@ export const search = action({ function publicSearchResult( r: Infer | null, - score: number + score: number, ): SearchResult | null { if (r === null) { return null; diff --git a/src/component/setup.test.ts b/src/component/setup.test.ts index def4cee..70a1a5c 100644 --- a/src/component/setup.test.ts +++ b/src/component/setup.test.ts @@ -3,17 +3,11 @@ import { test } from "vitest"; import { convexTest } from "convex-test"; import schema from "./schema.js"; export const modules = import.meta.glob("./**/*.*s"); - -// Sorry about everything -import componentSchema from "../../node_modules/@convex-dev/workpool/src/component/schema.js"; -export { componentSchema }; -export const componentModules = import.meta.glob( - "../../node_modules/@convex-dev/workpool/src/component/**/*.ts" -); +import workpool from "@convex-dev/workpool/test"; export function initConvexTest() { const t = convexTest(schema, modules); - t.registerComponent("workpool", componentSchema, componentModules); + t.registerComponent("workpool", workpool.schema, workpool.modules); return t; } diff --git a/src/shared.ts b/src/shared.ts index 8acda40..a77c84f 100644 --- a/src/shared.ts +++ b/src/shared.ts @@ -1,9 +1,14 @@ import { v } from "convex/values"; -import type { Infer, Validator, Value, VObject } from "convex/values"; +import type { + GenericId, + Infer, + Validator, + Value, + VObject, +} from "convex/values"; import { vNamedFilter, type NamedFilter } from "./component/filters.js"; import { brandedString } from "convex-helpers/validators"; import type { FunctionReference } from "convex/server"; -import { OpaqueIds } from "./client/types.js"; // A good middle-ground that has up to ~3MB if embeddings are 4096 (max). // Also a reasonable number of writes to the DB. @@ -22,7 +27,7 @@ export const vSearchResult = v.object({ v.object({ text: v.string(), metadata: v.optional(v.record(v.string(), v.any())), - }) + }), ), startOrder: v.number(), score: v.number(), @@ -33,7 +38,7 @@ export type SearchResult = Infer; export const vStatus = v.union( v.literal("pending"), v.literal("ready"), - v.literal("replaced") + v.literal("replaced"), ); export const vActiveStatus = v.union(v.literal("pending"), v.literal("ready")); export type Status = Infer; @@ -102,9 +107,7 @@ export type EntryFilter< }[keyof Filters & string]; export type Entry< - // eslint-disable-next-line @typescript-eslint/no-explicit-any Filters extends Record = any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any Metadata extends Record = any, > = { /** The entry's id, uniquely identifying the key + contents + namespace etc. */ @@ -166,8 +169,8 @@ export function vPaginationResult< v.union( v.literal("SplitRecommended"), v.literal("SplitRequired"), - v.null() - ) + v.null(), + ), ), }); } @@ -188,9 +191,7 @@ export const vOnCompleteArgs = v.object({ }); export type OnCompleteArgs< - // eslint-disable-next-line @typescript-eslint/no-explicit-any Filters extends Record = any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any EntryMetadata extends Record = any, > = { /** @@ -214,7 +215,7 @@ export type OnCompleteArgs< export type OnComplete = FunctionReference< "mutation", "internal", - OpaqueIds>, + IdsToStrings>, null >; @@ -227,7 +228,7 @@ export const vChunkerArgs = v.object({ export type ChunkerAction = FunctionReference< "action", "internal", - OpaqueIds>, + IdsToStrings>, null >; @@ -245,3 +246,12 @@ export function filterNamesContain(existing: string[], args: string[]) { } return true; } + +type IdsToStrings = + T extends GenericId + ? string + : T extends (infer U)[] + ? IdsToStrings[] + : T extends Record + ? { [K in keyof T]: IdsToStrings } + : T; diff --git a/src/test.ts b/src/test.ts index 2150188..6920008 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,11 +1,20 @@ +/// import type { TestConvex } from "convex-test"; import type { GenericSchema, SchemaDefinition } from "convex/server"; +import workpool from "@convex-dev/workpool/test"; import schema from "./component/schema.js"; const modules = import.meta.glob("./component/**/*.ts"); -function register( + +/** + * Register the component with the test convex instance. + * @param t - The test convex instance, e.g. from calling `convexTest`. + * @param name - The name of the component, as registered in convex.config.ts. + */ +export function register( t: TestConvex>, - name: string + name: string = "rag", ) { t.registerComponent(name, schema, modules); + workpool.register(t, "workpool"); } -export default { schema, modules, register }; +export default { register, schema, modules }; diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..aff5230 --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "src/**/*.tsx", + "example/src/**/*.ts", + "example/src/**/*.tsx", + "example/convex/**/*.ts" + ], + "exclude": ["node_modules", "dist", "**/_generated"] +} diff --git a/src/vitest.config.ts b/vitest.config.js similarity index 66% rename from src/vitest.config.ts rename to vitest.config.js index 28ce6fa..e9408f8 100644 --- a/src/vitest.config.ts +++ b/vitest.config.js @@ -3,5 +3,8 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "edge-runtime", + typecheck: { + tsconfig: "./tsconfig.test.json", + }, }, });