Upgrade redisvl and improve resource management

.gitignore

@@ -8,4 +8,5 @@ product_metadata.json
 product_vectors.json
 data/
 !backend/data
-.env
+.env
+.python-version

backend/productsearch/api/routes/product.py

@@ -2,10 +2,9 @@
 
 import numpy as np
 from fastapi import APIRouter, Depends
-from redis.commands.search.document import Document
 from redis.commands.search.query import Query
 from redisvl.index import AsyncSearchIndex
-from redisvl.query import FilterQuery, VectorQuery
+from redisvl.query import CountQuery, FilterQuery, VectorQuery
 from redisvl.query.filter import FilterExpression, Tag
 
 from productsearch import config
@@ -16,24 +15,9 @@
 )
 from productsearch.db import redis_helpers
 
-
 router = APIRouter()
 
 
-def create_count_query(filter_expression: FilterExpression) -> Query:
-    """
-    Create a "count" query where simply want to know how many records
-    match a particular filter expression
-
-    Args:
-        filter_expression (FilterExpression): The filter expression for the query.
-
-    Returns:
-        Query: The Redis query object.
-    """
-    return Query(str(filter_expression)).no_content().dialect(2)
-
-
 @router.get(
     "/",
     response_model=ProductSearchResponse,
@@ -116,14 +100,14 @@ async def find_products_by_image(
         return_fields=config.RETURN_FIELDS,
         filter_expression=filter_expression,
     )
-    count_query = create_count_query(filter_expression)
+    count_query = CountQuery(filter_expression)
 
     # Execute search
     count, result_papers = await asyncio.gather(
-        index.search(count_query), index.query(paper_similarity_query)
+        index.query(count_query), index.query(paper_similarity_query)
     )
     # Get Paper records of those results
-    return ProductVectorSearchResponse(total=count.total, products=result_papers)
+    return ProductVectorSearchResponse(total=count, products=result_papers)
 
 
 @router.post(
@@ -174,11 +158,11 @@ async def find_products_by_text(
         return_fields=config.RETURN_FIELDS,
         filter_expression=filter_expression,
     )
-    count_query = create_count_query(filter_expression)
+    count_query = CountQuery(filter_expression)
 
     # Execute search
     count, result_papers = await asyncio.gather(
-        index.search(count_query), index.query(paper_similarity_query)
+        index.query(count_query), index.query(paper_similarity_query)
     )
     # Get Paper records of those results
-    return ProductVectorSearchResponse(total=count.total, products=result_papers)
+    return ProductVectorSearchResponse(total=count, products=result_papers)

backend/productsearch/db/load.py

@@ -6,9 +6,10 @@
 
 import numpy as np
 import requests
-from productsearch import config
 from redisvl.index import AsyncSearchIndex
 
+from productsearch import config
+
 
 def read_from_s3():
     res = requests.get(config.S3_DATA_URL)
@@ -59,9 +60,9 @@ def preprocess(product: dict) -> dict:
 
 async def load_data():
     index = AsyncSearchIndex.from_yaml(
-        os.path.join("./productsearch/db/schema", "products.yml")
+        os.path.join("./productsearch/db/schema", "products.yml"),
+        redis_url=config.REDIS_URL,
     )
-    index.connect(config.REDIS_URL)
 
     # Check if index exists
     if await index.exists() and len((await index.search("*")).docs) > 0:

backend/productsearch/db/redis_helpers.py

@@ -1,10 +1,8 @@
 import logging
 import os
-from typing import List
 
 from redis.asyncio import Redis
 from redisvl.index import AsyncSearchIndex, SearchIndex
-from redisvl.query.filter import FilterExpression, Tag
 from redisvl.schema import IndexSchema
 
 from productsearch import config
@@ -15,13 +13,11 @@
 dir_path = os.path.dirname(os.path.realpath(__file__)) + "/schema"
 file_path = os.path.join(dir_path, "products.yml")
 schema = IndexSchema.from_yaml(file_path)
-client = Redis.from_url(config.REDIS_URL)
 global_index = None
 
 
 def get_test_index():
-    index = SearchIndex.from_yaml(file_path)
-    index.connect(redis_url=config.REDIS_URL)
+    index = SearchIndex.from_yaml(file_path, redis_url=config.REDIS_URL)
 
     if not index.exists():
         index.create(overwrite=True)
@@ -32,5 +28,5 @@ def get_test_index():
 async def get_async_index():
     global global_index
     if not global_index:
-        global_index = AsyncSearchIndex(schema, client)
+        global_index = AsyncSearchIndex(schema, redis_url=config.REDIS_URL)
     yield global_index

backend/pyproject.toml

@@ -1,23 +1,23 @@
 [tool.poetry]
 name = "productsearch"
-version = "0.1.0"
+version = "0.2.0"
 description = "Reference architecture for vector search application with Redis"
 authors = ["Robert Shelton  <robert.shelton@redis.com>"]
 readme = "README.md"
 package-mode = false
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = ">=3.11,<3.14"
 fastapi = "^0.111.0"
 uvicorn = "^0.30.1"
 ipython = "^8.26.0"
 numpy = "1.26.4"
-redisvl = "^0.2.3"
+redisvl = "^0.4.1"
 cohere = "^5.5.8"
 openai = "^1.35.9"
 sentence-transformers = "^3.0.1"
 sentencepiece = "^0.2.0"
-redis = "^5.0.7"
+
 
 [tool.poetry.group.dev.dependencies]
 mypy = "1.9.0"

docker-local-redis.yml

@@ -14,7 +14,7 @@ services:
     depends_on:
       - "redis-vector-db"
   redis-vector-db:
-    image: redis/redis-stack:latest
+    image: redis:8.0-M03
     ports:
       - "6379:6379"
       - "8001:8001"