diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5ebb76b8..78db4a34 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,15 +7,14 @@ on: pull_request: env: - WEAVIATE_124: 1.24.26 WEAVIATE_125: 1.25.34 WEAVIATE_126: 1.26.17 WEAVIATE_127: 1.27.27 WEAVIATE_128: 1.28.16 - WEAVIATE_129: 1.29.8 - WEAVIATE_130: 1.30.7 - WEAVIATE_131: 1.31.0 - WEAVIATE_132: 1.32.0-rc.1 + WEAVIATE_129: 1.29.9 + WEAVIATE_130: 1.30.12 + WEAVIATE_131: 1.31.5 + WEAVIATE_132: 1.32.4-cdf9a3b concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -43,7 +42,6 @@ jobs: fail-fast: false matrix: versions: [ - { node: "22.x", weaviate: $WEAVIATE_124}, { node: "22.x", weaviate: $WEAVIATE_125}, { node: "22.x", weaviate: $WEAVIATE_126}, { node: "22.x", weaviate: $WEAVIATE_127}, diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index a640dcbb..c77d4fd6 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -2,6 +2,7 @@ import { requireAtLeast } from '../../../test/version.js'; import { WeaviateUnsupportedFeatureError } from '../../errors.js'; import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js'; +import { WeaviateClass } from '../../openapi/types.js'; import { GenerativeCohereConfig, ModuleConfig, @@ -833,4 +834,47 @@ describe('Testing of the collection.config namespace', () => { expect(indexConfig.multiVector?.encoding).toBeUndefined(); } ); + + requireAtLeast(1, 32, 4).describe('uncompressed quantizer', () => { + it('should be able to create a collection with an uncompressed quantizer', async () => { + const collectionName = 'TestCollectionUncompressedVector'; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.selfProvided({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }); + await collection.config + .get() + .then((config) => + expect((config.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined() + ); + await fetch(`http://localhost:8080/v1/schema/${collectionName}`) + .then((res) => res.json() as WeaviateClass) + .then((schema) => + expect(schema.vectorConfig?.default.vectorIndexConfig?.skipDefaultQuantization).toBe(true) + ); + }); + + it('should be able to create a collection with uncompressed named vector', async () => { + const collectionName = 'TestCollectionUncompressedVectorNamed'; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.selfProvided({ + name: 'custom', + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }); + await collection.config + .get() + .then((config) => + expect((config.vectorizers.custom.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined() + ); + await fetch(`http://localhost:8080/v1/schema/${collectionName}`) + .then((res) => res.json() as WeaviateClass) + .then((schema) => + expect(schema.vectorConfig?.custom.vectorIndexConfig?.skipDefaultQuantization).toBe(true) + ); + }); + }); }); diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index b9c894cf..7d257048 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -68,6 +68,10 @@ export type RQConfig = { type: 'rq'; }; +export type UncompressedConfig = { + type: 'none'; +}; + export type MultiVectorConfig = { aggregation: 'maxSim' | string; encoding?: MultiVectorEncodingConfig; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 76104422..17b3dec5 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -219,6 +219,12 @@ export const parseVectorIndex = (module: ModuleConfig { diff --git a/src/collections/configure/parsing.ts b/src/collections/configure/parsing.ts index caeaaa3f..30b6b1e6 100644 --- a/src/collections/configure/parsing.ts +++ b/src/collections/configure/parsing.ts @@ -1,4 +1,4 @@ -import { MuveraEncodingConfigCreate } from '../index.js'; +import { MuveraEncodingConfigCreate, UncompressedConfigCreate } from '../index.js'; import { BQConfigCreate, BQConfigUpdate, @@ -49,6 +49,9 @@ export class QuantizerGuards { static isRQUpdate(config?: QuantizerConfig): config is RQConfigUpdate { return (config as RQConfigUpdate)?.type === 'rq'; } + static isUncompressedCreate(config?: QuantizerConfig): config is UncompressedConfigCreate { + return (config as UncompressedConfigCreate)?.type === 'none'; + } } type VectorIndexConfigCreate = diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index 4754f952..6009bbb6 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -2,15 +2,18 @@ import { WeaviateNestedProperty, WeaviateProperty } from '../../../openapi/types import { InvertedIndexConfig, MultiTenancyConfig, + QuantizerConfig, ReplicationConfig, ReplicationDeletionStrategy, } from '../../config/types/index.js'; -import { DataType } from '../../types/index.js'; +import { DataType, QuantizerRecursivePartial } from '../../types/index.js'; import { NonRefKeys, RefKeys } from '../../types/internal.js'; export type RecursivePartial = T extends object ? { - [P in keyof T]?: RecursivePartial; + [P in keyof T]?: T[P] extends QuantizerConfig + ? QuantizerRecursivePartial + : RecursivePartial; } : T; diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index b7c7624a..30423373 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -8,6 +8,7 @@ import { PQEncoderType, RQConfig, SQConfig, + UncompressedConfig, VectorDistance, VectorIndexConfigDynamic, VectorIndexConfigFlat, @@ -57,11 +58,14 @@ export type SQConfigUpdate = { type: 'sq'; }; +export type UncompressedConfigCreate = QuantizerRecursivePartial; + export type QuantizerConfigCreate = | PQConfigCreate | BQConfigCreate | SQConfigCreate | RQConfigCreate + | UncompressedConfigCreate | Record; export type QuantizerConfigUpdate = @@ -80,11 +84,23 @@ export type MuveraEncodingConfigCreate = RecursivePartial; export type MultiVectorEncodingConfigCreate = MuveraEncodingConfigCreate; -export type VectorIndexConfigHNSWCreate = RecursivePartial; +export type VectorIndexConfigHNSWCreate = RecursivePartial> & { + quantizer?: QuantizerConfigCreate; +}; -export type VectorIndexConfigDynamicCreate = RecursivePartial; +export type VectorIndexConfigDynamicCreate = RecursivePartial< + Omit +> & { + hnsw?: VectorIndexConfigHNSWCreate; + flat?: VectorIndexConfigFlatCreate; +}; -export type VectorIndexConfigDymamicUpdate = RecursivePartial; +export type VectorIndexConfigDymamicUpdate = RecursivePartial< + Omit +> & { + hnsw?: VectorIndexConfigHNSWUpdate; + flat?: VectorIndexConfigFlatUpdate; +}; export type VectorIndexConfigHNSWUpdate = { dynamicEfMin?: number; @@ -107,7 +123,9 @@ export type VectorIndexConfigCreateType = I extends 'hnsw' ? Record : never; -export type VectorIndexConfigFlatCreate = RecursivePartial; +export type VectorIndexConfigFlatCreate = RecursivePartial> & { + quantizer?: QuantizerConfigCreate; +}; export type VectorIndexConfigFlatUpdate = { quantizer?: BQConfigUpdate; diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 83f0eac9..17d9d64e 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -2,6 +2,7 @@ import { ModuleConfig, PQEncoderDistribution, PQEncoderType, + UncompressedConfig, VectorIndexFilterStrategy, } from '../config/types/index.js'; import { @@ -149,6 +150,16 @@ const configure = { * Define the quantizer configuration to use when creating a vector index. */ quantizer: { + /** + * Create an object of type `UncompressedConfig` to be used when defining the quantizer configuration of a vector index. + * + * This is useful for disabling the default quantization present in Weaviate>=1.33.0. + * + * @returns {UncompressedConfig} The object of type `UncompressedConfig`. + */ + none: (): UncompressedConfig => { + return { type: 'none' }; + }, /** * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index. * diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index b1a3d411..1fca4def 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -781,6 +781,8 @@ async function newClassObject(className: string, client: WeaviateClient): Promis vectorCacheMaxObjects: 500000, flatSearchCutoff: 40000, filterStrategy: (await isVer(client, 27, 0)) ? 'sweeping' : undefined, + skipDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined, + trackDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined, }, invertedIndexConfig: { cleanupIntervalSeconds: 60,