From 7cc33bcf608828493cc1bf68762806e17ab37b77 Mon Sep 17 00:00:00 2001 From: Robert Shelton Date: Thu, 26 Sep 2024 07:44:56 -0700 Subject: [PATCH 1/3] float16 recipe --- .../vector-search/02_float16_support.ipynb | 637 ++++++++++++++++++ 1 file changed, 637 insertions(+) create mode 100644 python-recipes/vector-search/02_float16_support.ipynb diff --git a/python-recipes/vector-search/02_float16_support.ipynb b/python-recipes/vector-search/02_float16_support.ipynb new file mode 100644 index 0000000..bddd147 --- /dev/null +++ b/python-recipes/vector-search/02_float16_support.ipynb @@ -0,0 +1,637 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'5.0.8'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import redis\n", + "redis.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![Redis](https://redis.io/wp-content/uploads/2024/04/Logotype.svg?auto=webp&quality=85,75&width=120)\n", + "# Using smaller vector types\n", + "\n", + "With the [Redis 7.4 release](https://redis.io/blog/announcing-redis-community-edition-and-redis-stack-74/) there is now support for bfloat16 and float16 data types in the vector store. Using this type is exactly the same as using float32 or other data types but will require a conversion if seeking to replace previously stored objects.\n", + "\n", + "This tutorial will walk through repopulating and index as such.\n", + "\n", + "## Packages" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "^C\n", + "\u001b[31mERROR: Operation cancelled by user\u001b[0m\u001b[31m\n", + "\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "# NBVAL_SKIP\n", + "%pip install -q redis redisvl numpy sentence-transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Redis Stack\n", + "\n", + "Later in this tutorial, Redis will be used to store, index, and query vector\n", + "embeddings created from PDF document chunks. **We need to make sure we have a Redis\n", + "instance available.\n", + "\n", + "#### For Colab\n", + "Use the shell script below to download, extract, and install [Redis Stack](https://redis.io/docs/getting-started/install-stack/) directly from the Redis package archive." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NBVAL_SKIP\n", + "%%sh\n", + "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", + "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", + "sudo apt-get update > /dev/null 2>&1\n", + "sudo apt-get install redis-stack-server > /dev/null 2>&1\n", + "redis-stack-server --daemonize yes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### For Alternative Environments\n", + "There are many ways to get the necessary redis-stack instance running\n", + "1. On cloud, deploy a [FREE instance of Redis in the cloud](https://redis.com/try-free/). Or, if you have your\n", + "own version of Redis Enterprise running, that works too!\n", + "2. Per OS, [see the docs](https://redis.io/docs/latest/operate/oss_and_stack/install/install-stack/)\n", + "3. With docker: `docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest`\n", + "\n", + "##### if running local version make sure to double check it is updated >=7.4\n", + "\n", + "### Define the Redis Connection URL\n", + "\n", + "By default this notebook connects to the local instance of Redis Stack. **If you have your own Redis Enterprise instance** - replace REDIS_PASSWORD, REDIS_HOST and REDIS_PORT values with your own." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Replace values below with your own if using Redis Cloud instance\n", + "REDIS_HOST = os.getenv(\"REDIS_HOST\", \"localhost\") # ex: \"redis-18374.c253.us-central1-1.gce.cloud.redislabs.com\"\n", + "REDIS_PORT = os.getenv(\"REDIS_PORT\", \"6379\") # ex: 18374\n", + "REDIS_PASSWORD = os.getenv(\"REDIS_PASSWORD\", \"\") # ex: \"1TNxTEdYRDgIDKM2gDfasupCADXXXX\"\n", + "\n", + "# If SSL is enabled on the endpoint, use rediss:// as the URL prefix\n", + "REDIS_URL = f\"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# from redis import Redis\n", + "import redis\n", + "\n", + "client = redis.Redis(\n", + " host='redis-11118.c309.us-east-2-1.ec2.redns.redis-cloud.com',\n", + " port=11118,\n", + " password='qt1Zl62JL07F7RTGac40XCiEhksh4xjD')\n", + "# client = Redis.from_url(REDIS_URL)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.ping()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Define index and load data\n", + "\n", + "We will be loading a schema of movie information with a 32bit float." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "with open(\"resources/movies.json\", 'r') as file:\n", + " movies = json.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "07:32:48 redisvl.index.index INFO Index already exists, overwriting.\n" + ] + } + ], + "source": [ + "from redisvl.schema import IndexSchema\n", + "from redisvl.index import SearchIndex\n", + "\n", + "index_name = \"movies32\"\n", + "\n", + "schema = IndexSchema.from_dict({\n", + " \"index\": {\n", + " \"name\": index_name,\n", + " },\n", + " \"fields\": [\n", + " {\n", + " \"name\": \"title\",\n", + " \"type\": \"text\",\n", + " },\n", + " {\n", + " \"name\": \"description\",\n", + " \"type\": \"text\",\n", + " },\n", + " {\n", + " \"name\": \"genre\",\n", + " \"type\": \"tag\",\n", + " \"attrs\": {\n", + " \"sortable\": True\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"rating\",\n", + " \"type\": \"numeric\",\n", + " \"attrs\": {\n", + " \"sortable\": True\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"vector\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 384,\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"hnsw\",\n", + " \"datatype\": \"float32\"\n", + " }\n", + " }\n", + " ]\n", + "})\n", + "\n", + "\n", + "index = SearchIndex(schema, client)\n", + "index.create(overwrite=True, drop=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Embed movie description vectors" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from sentence_transformers import SentenceTransformer\n", + "\n", + "# load model for embedding our movie descriptions\n", + "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n", + "\n", + "def embed_text(model, text):\n", + " return np.array(model.encode(text)).astype(np.float32).tobytes()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Note: convert embedding array to bytes for storage in Redis Hash data type\n", + "movie_data32 = [\n", + " {\n", + " **movie,\n", + " \"vector\": embed_text(model, movie[\"description\"])\n", + " } for movie in movies\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Populate the index" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['rvl:37b42d13adaa4a4ea32db5e4d062126b',\n", + " 'rvl:9e3dfc7c67b744bb8199d364c60f8d89',\n", + " 'rvl:8f7758c7763842f78490e3c43aab1148',\n", + " 'rvl:c961854993a04d089da2e026033b80b3',\n", + " 'rvl:7f2699e7dbf3474283d34bf04b465f00',\n", + " 'rvl:cc56f14da1f8431c90a318167ce7e506',\n", + " 'rvl:5c0097099939492f87b2b56890bc7dfe',\n", + " 'rvl:6c215979299c424ebff07a4b85a60f43',\n", + " 'rvl:896cd78dae42483db66019f2b62a70e4',\n", + " 'rvl:b92104d038c04ab3beab80139f75e7d1',\n", + " 'rvl:17b08c1a64b743ecab2e5acaa4a777af',\n", + " 'rvl:d7e8d94d158e4003ad443ec5ce517f53',\n", + " 'rvl:a448ffc20ac8474ab776df5e54ae26a9',\n", + " 'rvl:87e2cd98e776411cb7f552a29a808a2a',\n", + " 'rvl:750ccddcf5604b12b33fb81e78042db0',\n", + " 'rvl:403fcaf607b04e5a9b7561a86082cb26',\n", + " 'rvl:12e4333184554daca7cbbf44e77ccb18',\n", + " 'rvl:5be2ed277c1c4f3b8afb929446afda9e',\n", + " 'rvl:6ccdd8c9eb1f433fa92c85f16d4351ff',\n", + " 'rvl:7ba0ee30d58f45e7bc1d5d21fbe9a45d']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "keys = index.load(movie_data32)\n", + "keys" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "client.memory_usage('rvl:37f6b0ea780c4a84a156f35f54a31c7e')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Now let's convert to fp16\n", + "\n", + "This is as simple as you think it is using numpy." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "movie_data16 = []\n", + "for movie in movie_data32:\n", + " vect16 = np.frombuffer(movie[\"vector\"], dtype=np.float32).astype(np.float16)\n", + " movie[\"vector\"] = vect16.tobytes()\n", + " movie_data16.append(movie)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'title': 'Explosive Pursuit',\n", + " 'genre': 'action',\n", + " 'rating': 7,\n", + " 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.',\n", + " 'vector': b'\\xe3+S\\x18\\xbd\\x1d\\xf6\\xabs\\x9eQ*\\xfd)\\xac$8\\xb0{.\\x80+o\\xa8H\\xaa3&\\x0f+\\xb2\\xa0\\x9f \\xe5(O$n!\\xfe\\xa4\\x18\\xae\\x92*\\xbb\\xa7\\xba\\xb0)\\x88\\xfc%\\x17\\x93E\\xadg\\xa7+)\\x97\\x9b:*),\\x12\\xa5\\x83\\xaeY\\xaa\\xa7\\xa4G*1\"a\\'o\\xac\\xc3$\\x1d-\\x9b\\xa9\\xee\\xad\\xb9\\x1f\\xd8+f,0\\xae\\x82\\xac\\xe9\\x1dg\\xa8J\\xa6\\xe0\\xa5\\xa9\\x9bT&T*U)\\xaa\\xaa\\xb0\\xa4\\xc8(`\"N(d-\\xc7\\xad\\xe6\\xac\\xc4(3)\\xc2\\'H$\\x94\\x9f\\xda\\xaau\\x9f\\n\\xaa\\xcb\\xa8\\x93\\x9f\\xc6\\x1bz\"b\\xa1u-h\\xa1p\\xad\\xd7(n.\\x81)\\xf0\\xa4Z*\\x06)P\\xaa\\xf3\\x9c\\xbc\\x17\\xe8(\\xf6&\\xb3\\x98\\x83\\x9b\\xe7!\\x1d\\x85\\x9a#2!\\xc1 o\\xa9\\xc7$u*)-\\xf0&\\x9a p\\x96\\xe5\\xa6*(\\xad\\'\\x97\\xa2m\\xa1\\xe7\\nb/\\xb8*\\xb2\\xa6\\x99*\\xf7\\xaa\\x8d$\\x9f \\xce\\x9b|\\xa9\\x80#E\\x9c\\xef$\\xb0-\\x00\\x808\\x9d0\\xab[\\x9a\\x81\\x9c\\xcc\\x9d\\xce\\x9d[+h%C\\xadc.\\x07\\xb1z\\xa9\\xbb\\xa7\\xb5+\\x88#\\xe7\\xa8X$3\\xa54$\\xcc(%\\xa7\\xf3\\xa2s\\xa8\\x1e\\xa7\\xaa\\x1a\\xe2\\xab`\\xacT\\xa9\\xa4/\\x86\\x18q\\xa8\\xa7-\\xa9\\xa8\\x1c,\\xd1!\\x0f%h\\xacH\\xab\\xe8&\\xb4.\\xe5\\xac\\x0c\\xa9)\\x8e\\x9e\\xa4\\xf6\\xa91\\xaa\\xa2\\xb0D\\xa9J\\xac\\xa5&\\x19.x\\xaa\\x84\\xa3f\\xa5-\\xab\\x9c\\x1b\\x0b\\x1e\\x18\\xaa\\xf1\\x17\\xa7/Q!\\xe0(\\xb8\\xaao(7(\\xd4\\xad\\xb5&\\xcf\\x1ez).,\\x10){\\'\\x1f+B\\xb0\\xd9)\\xbb\\x1a\\x15#\\xbd%\\xdc\\xa9\\x8c\\xa4\\xe0\\xa9\\x19%*\\xa1\\xff\\x9d\\x94(\\xa9,\\xec\\xa7\\x90\\xae$\\x1c\\xa8+L#\\x95\\xaap\\xab\\x9e)7*\\x00\\x00\\xd2%}\\xaa8\\x1e(\\xa8\\x94\\xac\\'*\\xdf\\xa1\\xa2\\xa7\\xbc\\xack\\xa8\\xac\\xafA)6.\\xd7(\\xc5/v\\xa7\\xc1\\x98\\xdc!J\\xa8\\xb8(\\xf5(Z\\xaeh\\xack\\x9f\\xcb\\xa0\\x98\\xa0\\x1c,$\\xab\\x8e\\x9e\\xcb\\xa2\\x0c+\\xee\\xa7\\x1d\\x9c\\xca\\xa8\\xe8\\xaa\\xd7+\\xe2,K\\xad\\xcb\\xa1w\\xa2\\xc0\\xab\\x04-\\xb5$\\xc4\\x91\\xee\\xa9D!\\xa7\\x9d^\\xa8\\xa8\\xac\\xa6*\\x8c&:\\xa9@\"\\xc0%\\xeb\\xab\\x0c(\\xde\\xaa\\x1f {\\xa8<\\xa7A&\\xe0,7\\xad+/y\"\\x98)\\xb2\\xa9\\x97\\xa7;%\\x16\\xb0\\x93\\xa9T,N\\xa8n\\xa6o\\x9d\\xf8\\xa6)*\\xe8-i+s,%-U\\x9ej\\xa0\\xe2$s\\xacY+\\xf8\\x1d\\xf9\\'\\x92$b\\x1c\\x1d-\\xb1#\\x84\\x9f?\\xa0\\xed\\xa5\\x00\\x80\\x9c\\xaeW+?\\xad)\\'\\x81+\\xff,B\\xab\\xa5\\xaf?\\xa0W#\\xbc+\\x0f\\'\\xf5(\"&R #\\xa3h,4\\xa5i\\x96|.k\\xa9s+\\xf9\\xac\\xfc*\\x19!m\\x1b\\x04\\xb0\\\\\\xa4;*U.\\x8e\\xa3\\xd7\\xa9\\xed\"\\x8b\\x1es-\\x19\\xa8J,\\x9a\\xab\\x11+N\\x98@\\'&\\xa7\\x1b)\\\\\\xab\\x0b w\\xaa\\x88\\xa5P\\xaa\\xf8+\\xd7\\xaa\\x89\\x9f\\xc2)\\xa2\\xa2\\t$m*/ \\xe2\\xab\\xbd#?\\xad\\x03\\xa99(\\xa1\\xa0L\\'\\xb9,'}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "movie_data16[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create new index" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "ename": "ValidationError", + "evalue": "1 validation error for IndexSchema\n__root__ -> attrs -> datatype\n value is not a valid enumeration member; permitted: 'FLOAT32', 'FLOAT64' (type=type_error.enum; enum_values=[, ])", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[12], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mredisvl\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mindex\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SearchIndex\n\u001b[1;32m 4\u001b[0m index_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmovies16\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 6\u001b[0m schema \u001b[38;5;241m=\u001b[39m \u001b[43mIndexSchema\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mindex\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfields\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtitle\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 15\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdescription\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 18\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 19\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 20\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgenre\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtag\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 22\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msortable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 24\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 25\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 26\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrating\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 28\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnumeric\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 29\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 30\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msortable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 31\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 32\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 33\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 34\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvector\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 35\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvector\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdims\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m384\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdistance_metric\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcosine\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43malgorithm\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhnsw\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdatatype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfloat16\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\n\u001b[1;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 44\u001b[0m \u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m index \u001b[38;5;241m=\u001b[39m SearchIndex(schema, client)\n\u001b[1;32m 48\u001b[0m index\u001b[38;5;241m.\u001b[39mcreate(overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "File \u001b[0;32m~/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/schema/schema.py:263\u001b[0m, in \u001b[0;36mIndexSchema.from_dict\u001b[0;34m(cls, data)\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[1;32m 228\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfrom_dict\u001b[39m(\u001b[38;5;28mcls\u001b[39m, data: Dict[\u001b[38;5;28mstr\u001b[39m, Any]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIndexSchema\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 229\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Create an IndexSchema from a dictionary.\u001b[39;00m\n\u001b[1;32m 230\u001b[0m \n\u001b[1;32m 231\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124;03m })\u001b[39;00m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.11.9/lib/python3.11/site-packages/pydantic/v1/main.py:341\u001b[0m, in \u001b[0;36mBaseModel.__init__\u001b[0;34m(__pydantic_self__, **data)\u001b[0m\n\u001b[1;32m 339\u001b[0m values, fields_set, validation_error \u001b[38;5;241m=\u001b[39m validate_model(__pydantic_self__\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, data)\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validation_error:\n\u001b[0;32m--> 341\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m validation_error\n\u001b[1;32m 342\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 343\u001b[0m object_setattr(__pydantic_self__, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__dict__\u001b[39m\u001b[38;5;124m'\u001b[39m, values)\n", + "\u001b[0;31mValidationError\u001b[0m: 1 validation error for IndexSchema\n__root__ -> attrs -> datatype\n value is not a valid enumeration member; permitted: 'FLOAT32', 'FLOAT64' (type=type_error.enum; enum_values=[, ])" + ] + } + ], + "source": [ + "from redisvl.schema import IndexSchema\n", + "from redisvl.index import SearchIndex\n", + "\n", + "index_name = \"movies16\"\n", + "\n", + "# this won't work till we merge Justin's stuff\n", + "schema = IndexSchema.from_dict({\n", + " \"index\": {\n", + " \"name\": index_name,\n", + " },\n", + " \"fields\": [\n", + " {\n", + " \"name\": \"title\",\n", + " \"type\": \"text\",\n", + " },\n", + " {\n", + " \"name\": \"description\",\n", + " \"type\": \"text\",\n", + " },\n", + " {\n", + " \"name\": \"genre\",\n", + " \"type\": \"tag\",\n", + " \"attrs\": {\n", + " \"sortable\": True\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"rating\",\n", + " \"type\": \"numeric\",\n", + " \"attrs\": {\n", + " \"sortable\": True\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"vector\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 384,\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"hnsw\",\n", + " \"datatype\": \"float16\"\n", + " }\n", + " }\n", + " ]\n", + "})\n", + "\n", + "\n", + "index = SearchIndex(schema, client)\n", + "index.create(overwrite=True, drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from redis.commands.search.field import VectorField, TagField, NumericField, TextField\n", + "from redis.commands.search.indexDefinition import IndexDefinition, IndexType\n", + "\n", + "index_name = \"movies\"\n", + "\n", + "schema = (\n", + " VectorField(\n", + " \"vector\",\n", + " \"HNSW\",\n", + " {\n", + " \"TYPE\": \"FLOAT16\",\n", + " \"DIM\": 384,\n", + " \"DISTANCE_METRIC\": \"COSINE\"\n", + " }\n", + " ),\n", + " NumericField(\"rating\"),\n", + " TagField(\"genre\"),\n", + " TextField(\"title\"),\n", + " TextField(\"description\")\n", + ")\n", + "\n", + "try:\n", + " client.ft(index_name).info()\n", + " print(\"Index exists!\")\n", + "except:\n", + " # index Definition\n", + " definition = IndexDefinition(index_type=IndexType.HASH)\n", + "\n", + " # create Index\n", + " client.ft(index_name).create_index(fields=schema, definition=definition)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def load_docs(client: redis.Redis, data: list[dict]):\n", + " for i, d in enumerate(data):\n", + " client.hset(\n", + " i,\n", + " mapping = d\n", + " )\n", + "\n", + "def print_results(res):\n", + " docs = [(doc.title, doc.genre, doc.rating) for doc in res.docs]\n", + " print(f\"Top {len(docs)} movies: \", docs)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "load_docs(client, movie_data16)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Result{20 total, docs: [Document {'id': '0', 'payload': None, 'title': 'Explosive Pursuit', 'genre': 'action', 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.', 'vector': '+S\\x18\\x1dsQ*)$8{.+oH3&\\x0f+ (O$n!\\x18*)%\\x17Eg+):*),\\x12YG*1\"a\\'o$\\x1d-\\ueb79\\x1f+f,0\\x1dgJ३T&T*U)(`\"N(d-ǭ(3)\\'H$ڪu\\n˨\\x1bz\"bu-hp(n.)Z*\\x06)P\\x17(&!\\x1d#2! o$u*)-& p*(\\'m\\nb/**$ Λ|#E$-\\x0080[̝Ν[+h%Cc.\\x07z+#X$34$(%s\\x1e\\x1a`T/\\x18q-\\x1c,!\\x0f%hH&.\\x0c)1DJ&\\x19.xf-\\x1b\\x0b\\x1e\\x18\\x17/Q!(o(7(ԭ&\\x1ez).,\\x10){\\'\\x1f+B)\\x1a\\x15#%ܩ\\x19%*(,짐$\\x1c+L#p)7*\\x00\\x00%}8\\x1e(\\'*ߡkA)6.(/v!J((Zhkˠ\\x1c,$ˢ\\x0c+\\x1dʨ+,Kˡw\\x04-$đD!^*&:@\"%\\x0c(ު\\x1f {\\x07(.\\x1c@.\\x1e[)߬,\"5(\\x1f([\\x00|7\\x1e+19A-\\x16%\\x1f$\\x08,KX\\x1d*.m-ۢ寋\\x1f)\\x17v(*\"3%N)&\\x1f\\x1f\\x02\\x15\\x04\\x16,\\x9ey!Z+\\x12|\\x08$Uz%\\x1cRh\\x0c\\x18(#N\\x1dGJdÕX\\x0c\\x19蜗./\\x1b\"̘%)b+**.Ϫ\\x00 @,_-\\x042y+.1\\x1fS{)d\\x00\\x00o\\x1eɪ:\\x7f,\\x08w\"] \\x1c.W\\'!\\x00)Ѧ\\x06\\'\\x04-\\x1b\\x17.L %\\x1c,6$+M\\x1du\\x1f\\'e0#*(z)V,$\\x02ܬ\"&p%\\r2\\x1f-D\\'t*9\\x03ά%˩%$\\x1aW)%,<\\x15,!%\\x1f!k\\x1e&\\x16*)\\x17\\x1e((u*(>\\x03[\\x1cc,%\\x003p(w*\\x16\\x1f4 =)+]\\x14\\x18\\x1a#\\x1e,G\\x1f!\\'ڦ\\',\\x1c\\x1f)\\t$j/)N)\\x16(\\x0co+t? \\x06.-ר+\\x18,\\x18ԫf*O)\\n kũ`*)\\x0f\\'m\\\\%B#\\x17)', 'rating': '8'}, Document {'id': '2', 'payload': None, 'title': 'Fast & Furious 9', 'genre': 'action', 'description': 'Dom and his crew face off against a high-tech enemy with advanced weapons and technology.', 'vector': '=\\x11-A\\x1d*+\\x01-g4);(m+\\x1c\\rT,\\x1e+b#\\x1bء(\\x18\\x00.q-uڥL(֦\\x08>}-\\x1d*O¬P \\x18\\x05)J\\x1a,\\x1b*!)\\x19&\\x14R,A#Wo&ʡA+\\r+),-\\x1dU#ߤ&&\\x1fᗬ/۩?*3˕~+Ȩ._)s&\\x02.\\x1c\\x11-,$%<+)\\x04\\'(,d$1g3*![%8\\t5)V1(LӨ|.-*\\x00\\x19\"Z+(* c-\\x19ǩ$A\\x1d\\x1c*ޣ\\t\\x16M\\'#E%,l\\x155\\x1f)-\\x00\\x15,\\x10*1-(ͥȫj\\x11I$\\x13\\'\"+\\x19%˨#\\'\\'\\'\\u2438-(**,*˦[t)r\\x02)H*U(먤%p\"$#$\\x1b$\\x0eb\\'AC)?\\x1aD\\x14dާ\\x00\\x008b& u*z\\x1f,+],)Bp5r,\\x0c\\x12,S\\x1c(1y\\',)Z*p(\\x19+\\x07$e 0*ө_\\x0b,Ĥ~7.vQ\\x1c\\x15&~) c,G,\\x1b!;*\\x07\\x1d\\x10(J-\\x1b,/.ά*ik) \\x1cp*k(\\x1e¨-)c/ɝ\\x12\\\\w{\\x1f\\x1c9\\x00dG\\x01\\x1f&۩h,A\\x1d$ -h\\'R+,\\x0bcԮ\\x08\\x1d).(o%-z&,-; \\x1c0\\x13*Μ,ҭ*H\\'v,K-\\x90\\x02\\x13\\x1d\\x04)$-Ir$B 9\\x15,R(', 'rating': '6'}, Document {'id': '3', 'payload': None, 'title': 'Black Widow', 'genre': 'action', 'description': 'Natasha Romanoff confronts her dark past and family ties as she battles a new enemy.', 'vector': '/~\\x17+%\\x04-Z(Δ\\x1f?\\x1c\\x16$6:B\\x14$\\x1c\\r\\x1ff\\x0b#)\\x06\\'(6\\x1c&\\x13),\"&$*-y)&)E*R\\x00u\\x16$.\\x1d\\x07\\x14V\\x1dXn)-.)\\x1d\\'\\'*]J\\x13)\\'%{,Y\\x07*+D˪\\x11..(%\\x15))\"M!(\\x17\\n!R+\\x02E\\x1f ^,\\'&Ց$\\x10**\\x0f1(ڨ)rڬ)+&\\x17.\\x01 /\\x18oK,J)[*\\x06%T\\x1f8\\x00N#%K\\x16/(!)r(}\\x1eV,x`)d*j(#/z\\x1d/0\\x13?ӧ\\n(6(}$*%\\'-$d*\\'(\\x06\\x14\\x1e\\x13`(JY4#v,4T!<\\x02\\x00*$\\x17-\\'i5*\\x14+\\x18\\x19ҥi&($.\\x1c t\\x1a!)\\x0f,6۬7\\'A*\\x04\\x1f\\x08(\\x02\\x05)|&\\x00-k\\x7f&,⡡X.P!\\x0b)!(2(#Ѫ.\\x1e\\x94\\x10\\x1a\\x06`,j0}\\x15oG\\x06\\x1b\\'*0*ˮ\\'$8\\x0b-?-m/u)0\\x0eXt*~,ί)\\x1f,/D\\x10%v\\x00zl\\x1f٩j-̦2\\x1b,]$\\x0b\\'$)F\\',\\x13/&}.E(gK\\x14L\\x00硇R\\'Ȭժ+\\x1bb\\x1a\\x1d*\\x05\"j)#Ϊ1.k-7we\\x13\\x05\\'g\\x15rΥ\\t,)%!Ҩy-%f\\x1fwg*P,hRw\"*+1<\\x18\\x1c\\x1a)m(8)\"\\x0c*%,h*U', 'rating': '7'}, Document {'id': '4', 'payload': None, 'title': 'John Wick', 'genre': 'action', 'description': 'A retired hitman seeks vengeance against those who wronged him, leaving a trail of destruction in his wake.', 'vector': '\\x9b).\\x19$\\x1b\\r#Mդk\\'\\x03H(p\\x1f%2$.l$@:-/\\x11QSѥ\\x03*%թţ\\x1cޭO$*H#%%h$ʩ\\x0f!C+\\x00*fa((h(kx-y\\x1ch*ĢQ()d%\\\\,U(j,)E$\\x0e|$%,Q!^-* #*K+)$%1\\x10,˧*k.؟l5#()$ѯ\\x15,\\x0eϘd(\\x1c,窺;,4(!k\\x1d\\x19,\\x1d\\x00(&& \\x00y,#\\n\\'w%\\x16&a)&*y\\x11/$\"?#,ެP\\x08.ǥ(\\x17g#(5xi/,U*!#a(u(\\x0c\\x16\\x19T$.h1!0%%w\\x01,[\">N\\'A(4\\x19+\\x03Y\\x1f.\"\\x1e\\n:\\x1d-<\\x1e)\\x00P)D*ڦ\\x15%&ʨե&\\'\\x05\\x01\\t&+$$p,=\\x00\\x18#\\')\\x03,\\x1e,,t4>1/\\x1f#\\x10QM\"=-\\x0b̟],2\\x0b#)H-9.& V#h\\x1e\\'$\\x02$+0\\x16+.\\x06)\\'\\'-\\'d*E\\x1eŬ \\x061\\x14R)ƩVW*\\x10XI&ͫ\\x04\"#\\'ҭ\\x17,\\x0f\\x1fB)#\\'!ӟ\\x00\\x04ꕔ%\\x12 +!]m*\\x19%o--s\\x1e$(Nάܪ\\x1cq\"(,N*&\\x1eB) \\x0e(y%p)+\\x10?A,(\\'3ǮW $wt)+\"\\x0c)t)yY(!Ƥ0\\x0536\"', 'rating': '8'}, Document {'id': '6', 'payload': None, 'title': 'The Dark Knight', 'genre': 'action', 'description': 'Batman faces off against the Joker, a criminal mastermind who threatens to plunge Gotham into chaos.', 'vector': '\\n/)j &*,\\'l$rR(\\x1d;ѥZ%, (*]-|\\x1f\\x03k+v(+$\\x1d)֭(6#բ0E+0() =&$9Y+*}\\x12+m(\\x1c(\\x11$I!V.()\"\\'l)J),\\x12*O Р,V\\'X$Ϥ&\\'\\x1a,\\x11\\')\\'N*ʠ1\\'\\x1c-::DJ)%\\'}(4\\x1fg\\'+S\\x08\\\\\\x07-\\x1f\\x10 !,*L)0**d),\\n--G!\\x15O)[Ȫ\\x00u\\x05\\x11T,fS%8)0!ps0e\\x07,\\x0c\\'7\\x0e\\x19g\\x08 g%\\x04\"(l*#R-c_)G)*\\x1adĭM\\x15̨߬#y ($쬮#9~)S\\x7f\\x0b#*\\x1fr\\x1fǡ5T\"+\\x19\"4\\'\\x17),\\x7fQ*d)\\t/T\\n\\x1c#ҭ¬\\r%\\x19s)릶(j*\\x00&@q9#\\x1dʬ*Y\\x1f8,(,*+\\x14)H&Eȡ\\x01\\n\\x17*#!\\t*l&(&((0gJ+\\x02)x\\x1c*\\x0f+Z.<(\\r(a$\\n( r`\\x1d}\\x11]1\\')N*\\x14%u+d(/\\x18\\x1fz\\x07ު\\r)!דɧ$+?(\\x10.\\x1f\\x12*\\x07*\\x05+%\\x00T˪]!\\x03$s+;Il*x%t% ,Z\\x06\\x0f=\\x18ȮC >\\',*ߣ3,\\x1a/*R$(t,ǩ\\\\$*~(;+2K& \\x1c\"uT\\x1d\\\\\\x11)Pfz$M)\\x15,\\x0b*b\\x1e', 'rating': '9'}, Document {'id': '7', 'payload': None, 'title': 'Gladiator', 'genre': 'action', 'description': 'A betrayed Roman general seeks revenge against the corrupt emperor who murdered his family.', 'vector': '.+\\x93;m^)\\x7f*\\x19\\x1a*3#\\x01\\x02.~$\\x02+$\\x1f|)e+X\\x04\\x04(C7,p\\x05\\x05M|+%.\\'*.50\\'~+(*$\\'t&\\x1a\\x11.\\x01\\x12\\'Ц+ (,\\x13\\'\\x1aWX\\';? \\x1c/Z+x\\x1d\\x1c\\x1a)\\x0f%jqT\\x14s*7\\x1e[*Fo,\\x0c0*hW*̠]-++h(o\\x1ebh)*\\x16\\x003\\x1clb!)*\\x0b\\x1dy.\\x1f@+,vV.,\\x197&(%[\\x1dQ(&\\'-(I#E!0$ЬU.X(-t(\\x1c&z$jED\\x1c(,$;\\x1cP*))\"),\\',(`^a$M5,\\x13\\x18a|(W9Ԯ\\x1e#i$\"\\x9d\\x08ޙ*\\x03!sì.l\\x00S\\x1f\\n$*\\x07!2k7\\'\\x1a$],V(ɪP0%##\\x1fj&\"z\\x0f).((,\\x0f,P#\\x01(G\\x1c./,\\x0bҡ$,\\'j,-˦)\\x08&h\\')&.\\x08.k(.v$̦\\'T\\x10$-Q(:r!(\\x07%O\\x1e\\'!,f-)*A\\x0f)B%t*}\\x0f,\\n.\\x1b-c\\x0c( .})\\x00\\x0c9&7\\x0f֬\\x1b\\x1f,!+\\r,\\x16%\\x1e\\\\\\x1cX\\x16\\x0f\\'ĥ\\x00-G\"\\x1e٥V, *̠!U,|+X\\',&m\\x02\\x01*,ʤQ\\x1d%,l\\x1a\\x15o*\\x12.(w#e\\x14', 'rating': '8'}, Document {'id': '8', 'payload': None, 'title': 'Inception', 'genre': 'action', 'description': 'A thief who enters dreams to steal secrets faces his toughest mission yet, with reality itself at stake.', 'vector': '**6$\\x18!\"&T+\\x1d١.$\"\\x0e3(%\\x01&*1\\x02**d\\')Щ\\uf805\\x1a-1*%+7էC \\x1c+ܡߡ\\x0f$\\x07**$~\\x1d(+ܫw($)[-\\x17\\x04+\\x0f(G2X-\\x1a+ףsp$%,hIf\\'\\x1cOg!-LA\\x10\\x0b#;\\n\\x03XR&,N-+Jx\\x1f;\\x00ܠ*!?#d(a&iAw\\x15Q.|\\'4*_8,a\\x14/Q8\\x0c*E9\\x1dW)m\\x1b\\x1d,N*&*-K*Wjh2\\'\\x11\\x15g&(\\x10)(\\x1aR\\x16\\x14\\x11),\\x00)9-\\x19*\\x17\\x19#-˨4\\x1aK\\x07ΩݦO&\\x02,\\x1f\\x0b7\\x05x(!\"-g%q\\x03@)\\':\\x07$\\x1d\\x00j-\\x16x\\'r,\\t,7\"\\x1e}*%)/\\r\\n*\\x06q #((\\n(N,o\\x16NE\\x1f߭M%aO-I[\\'y\\x11ħh\\\\\\\\,\\x1c,,\\x00-h)F)\\x03$\\x1b&s\\x1b()@+n%R\")(%`!.\\r)6-\\x11B&.!A((\\x13\\x1c,%J>*N(.ɫ\\x1b\\x08t$S\\x00{2+n*\\x02&W1K,*p\\'g)\\x19\\x18Ț)#-\"~\\'m# X0[&ɪd(\\x0e%6+)D&\\x13,\\x1d(,(\\x7f䧨,ʫ+\\x1eS\\'?\\x18\\x04-,m$M#4(^[\\x1c+\\x05\"\\x15\\x01', 'rating': '9'}, Document {'id': '9', 'payload': None, 'title': 'The Avengers', 'genre': 'action', 'description': \"Earth's mightiest heroes come together to stop an alien invasion that threatens the entire planet.\", 'vector': '&\\')\\x1b-0¨ȧ,=)\\x1c㞌)O\\x18\\'yu/4(-g+P!c\\x12\\x1dԧt$1\\x03*| m*\\x0fX&Ȭ[\\x1e\\x04\\x1cŬaC#p\\'Ϣ߫,w**\\x14ᠸ\\x1f$˥\\x1ff\\x1b*/\\x0c)\"\\x15<\\x14zc+,9%\\',[]&a\\x16@)4+$`-C+*ӣҪ&: w\\x195F&p \\x7f\\x1d>*0x*B\\x1d`**,)0\\x04\\t*$\\x07)|,x\\x12\\x00G\\x1b\\x1f(\\x1e&l\\x04-)C#_=)\\x0e(\\x1e$i(6\"C4\\x7f+\\x11(\\x03#!,Ξ\\x13J,Q-\"$&?(h+T,S\\x0c,j$d&+XT!f!\\x1b\\'\\x0c\\x1b%\\x18*\\x05+X)\\x1cΰ-%@\\x1f{--04\\t&$(\"X$w)\\x1cy zZ)y$)\\x02K\\x00\\x00˨\\x04\\x1a)\\x00/+)kr*\\x0f,\\x14?,:/\"M&`-(#\\ue840,#((@(\\x160\\x00-e!}C>\"()\\'\\x05+\\',\\x1d,\\x04w),m(a=+\\x1a-*\\'X,(\\x16ק!a,\\x18*+៥ڨs)\\x15?u\\x13$l\\x00~\\x1c,HX)(', 'rating': '8'}]}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "res = client.ft(index_name).search(\"*\")\n", + "res" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document {'id': '0', 'payload': None, 'title': 'Explosive Pursuit', 'genre': 'action', 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.', 'vector': '+S\\x18\\x1dsQ*)$8{.+oH3&\\x0f+ (O$n!\\x18*)%\\x17Eg+):*),\\x12YG*1\"a\\'o$\\x1d-\\ueb79\\x1f+f,0\\x1dgJ३T&T*U)(`\"N(d-ǭ(3)\\'H$ڪu\\n˨\\x1bz\"bu-hp(n.)Z*\\x06)P\\x17(&!\\x1d#2! o$u*)-& p*(\\'m\\nb/**$ Λ|#E$-\\x0080[̝Ν[+h%Cc.\\x07z+#X$34$(%s\\x1e\\x1a`T/\\x18q-\\x1c,!\\x0f%hH&.\\x0c)1DJ&\\x19.xf-\\x1b\\x0b\\x1e\\x18\\x17/Q!(o(7(ԭ&\\x1ez).,\\x10){\\'\\x1f+B)\\x1a\\x15#%ܩ\\x19%*(,짐$\\x1c+L#p)7*\\x00\\x00%}8\\x1e(\\'*ߡkA)6.(/v!J((Zhkˠ\\x1c,$ˢ\\x0c+\\x1dʨ+,Kˡw\\x04-$đD!^*&:@\"%\\x0c(ު\\x1f {\\x07(.\\x1c@.\\x1e[)߬,\"5(\\x1f([\\x00|7\\x1e+19A-\\x16%\\x1f$\\x08,KX\\x1d*.m-ۢ寋\\x1f)\\x17v(*\"3%N)&\\x1f\\x1f\\x02\\x15\\x04\\x16,\\x9ey!Z+\\x12|\\x08$Uz%\\x1cRh\\x0c\\x18(#N\\x1dGJdÕX\\x0c\\x19蜗./\\x1b\"̘%)b+**.Ϫ\\x00 @,_-\\x042y+.1\\x1fS{)d\\x00\\x00o\\x1eɪ:\\x7f,\\x08w\"] \\x1c.W\\'!\\x00)Ѧ\\x06\\'\\x04-\\x1b\\x17.L %\\x1c,6$+M\\x1du\\x1f\\'e0#*(z)V,$\\x02ܬ\"&p%\\r2\\x1f-D\\'t*9\\x03ά%˩%$\\x1aW)%,<\\x15,!%\\x1f!k\\x1e&\\x16*)\\x17\\x1e((u*(>\\x03[\\x1cc,%\\x003p(w*\\x16\\x1f4 =)+]\\x14\\x18\\x1a#\\x1e,G\\x1f!\\'ڦ\\',\\x1c\\x1f)\\t$j/)N)\\x16(\\x0co+t? \\x06.-ר+\\x18,\\x18ԫf*O)\\n kũ`*)\\x0f\\'m\\\\%B#\\x17)', 'rating': '8'},\n", + " Document {'id': '2', 'payload': None, 'title': 'Fast & Furious 9', 'genre': 'action', 'description': 'Dom and his crew face off against a high-tech enemy with advanced weapons and technology.', 'vector': '=\\x11-A\\x1d*+\\x01-g4);(m+\\x1c\\rT,\\x1e+b#\\x1bء(\\x18\\x00.q-uڥL(֦\\x08>}-\\x1d*O¬P \\x18\\x05)J\\x1a,\\x1b*!)\\x19&\\x14R,A#Wo&ʡA+\\r+),-\\x1dU#ߤ&&\\x1fᗬ/۩?*3˕~+Ȩ._)s&\\x02.\\x1c\\x11-,$%<+)\\x04\\'(,d$1g3*![%8\\t5)V1(LӨ|.-*\\x00\\x19\"Z+(* c-\\x19ǩ$A\\x1d\\x1c*ޣ\\t\\x16M\\'#E%,l\\x155\\x1f)-\\x00\\x15,\\x10*1-(ͥȫj\\x11I$\\x13\\'\"+\\x19%˨#\\'\\'\\'\\u2438-(**,*˦[t)r\\x02)H*U(먤%p\"$#$\\x1b$\\x0eb\\'AC)?\\x1aD\\x14dާ\\x00\\x008b& u*z\\x1f,+],)Bp5r,\\x0c\\x12,S\\x1c(1y\\',)Z*p(\\x19+\\x07$e 0*ө_\\x0b,Ĥ~7.vQ\\x1c\\x15&~) c,G,\\x1b!;*\\x07\\x1d\\x10(J-\\x1b,/.ά*ik) \\x1cp*k(\\x1e¨-)c/ɝ\\x12\\\\w{\\x1f\\x1c9\\x00dG\\x01\\x1f&۩h,A\\x1d$ -h\\'R+,\\x0bcԮ\\x08\\x1d).(o%-z&,-; \\x1c0\\x13*Μ,ҭ*H\\'v,K-\\x90\\x02\\x13\\x1d\\x04)$-Ir$B 9\\x15,R(', 'rating': '6'},\n", + " Document {'id': '3', 'payload': None, 'title': 'Black Widow', 'genre': 'action', 'description': 'Natasha Romanoff confronts her dark past and family ties as she battles a new enemy.', 'vector': '/~\\x17+%\\x04-Z(Δ\\x1f?\\x1c\\x16$6:B\\x14$\\x1c\\r\\x1ff\\x0b#)\\x06\\'(6\\x1c&\\x13),\"&$*-y)&)E*R\\x00u\\x16$.\\x1d\\x07\\x14V\\x1dXn)-.)\\x1d\\'\\'*]J\\x13)\\'%{,Y\\x07*+D˪\\x11..(%\\x15))\"M!(\\x17\\n!R+\\x02E\\x1f ^,\\'&Ց$\\x10**\\x0f1(ڨ)rڬ)+&\\x17.\\x01 /\\x18oK,J)[*\\x06%T\\x1f8\\x00N#%K\\x16/(!)r(}\\x1eV,x`)d*j(#/z\\x1d/0\\x13?ӧ\\n(6(}$*%\\'-$d*\\'(\\x06\\x14\\x1e\\x13`(JY4#v,4T!<\\x02\\x00*$\\x17-\\'i5*\\x14+\\x18\\x19ҥi&($.\\x1c t\\x1a!)\\x0f,6۬7\\'A*\\x04\\x1f\\x08(\\x02\\x05)|&\\x00-k\\x7f&,⡡X.P!\\x0b)!(2(#Ѫ.\\x1e\\x94\\x10\\x1a\\x06`,j0}\\x15oG\\x06\\x1b\\'*0*ˮ\\'$8\\x0b-?-m/u)0\\x0eXt*~,ί)\\x1f,/D\\x10%v\\x00zl\\x1f٩j-̦2\\x1b,]$\\x0b\\'$)F\\',\\x13/&}.E(gK\\x14L\\x00硇R\\'Ȭժ+\\x1bb\\x1a\\x1d*\\x05\"j)#Ϊ1.k-7we\\x13\\x05\\'g\\x15rΥ\\t,)%!Ҩy-%f\\x1fwg*P,hRw\"*+1<\\x18\\x1c\\x1a)m(8)\"\\x0c*%,h*U', 'rating': '7'},\n", + " Document {'id': '4', 'payload': None, 'title': 'John Wick', 'genre': 'action', 'description': 'A retired hitman seeks vengeance against those who wronged him, leaving a trail of destruction in his wake.', 'vector': '\\x9b).\\x19$\\x1b\\r#Mդk\\'\\x03H(p\\x1f%2$.l$@:-/\\x11QSѥ\\x03*%թţ\\x1cޭO$*H#%%h$ʩ\\x0f!C+\\x00*fa((h(kx-y\\x1ch*ĢQ()d%\\\\,U(j,)E$\\x0e|$%,Q!^-* #*K+)$%1\\x10,˧*k.؟l5#()$ѯ\\x15,\\x0eϘd(\\x1c,窺;,4(!k\\x1d\\x19,\\x1d\\x00(&& \\x00y,#\\n\\'w%\\x16&a)&*y\\x11/$\"?#,ެP\\x08.ǥ(\\x17g#(5xi/,U*!#a(u(\\x0c\\x16\\x19T$.h1!0%%w\\x01,[\">N\\'A(4\\x19+\\x03Y\\x1f.\"\\x1e\\n:\\x1d-<\\x1e)\\x00P)D*ڦ\\x15%&ʨե&\\'\\x05\\x01\\t&+$$p,=\\x00\\x18#\\')\\x03,\\x1e,,t4>1/\\x1f#\\x10QM\"=-\\x0b̟],2\\x0b#)H-9.& V#h\\x1e\\'$\\x02$+0\\x16+.\\x06)\\'\\'-\\'d*E\\x1eŬ \\x061\\x14R)ƩVW*\\x10XI&ͫ\\x04\"#\\'ҭ\\x17,\\x0f\\x1fB)#\\'!ӟ\\x00\\x04ꕔ%\\x12 +!]m*\\x19%o--s\\x1e$(Nάܪ\\x1cq\"(,N*&\\x1eB) \\x0e(y%p)+\\x10?A,(\\'3ǮW $wt)+\"\\x0c)t)yY(!Ƥ0\\x0536\"', 'rating': '8'},\n", + " Document {'id': '6', 'payload': None, 'title': 'The Dark Knight', 'genre': 'action', 'description': 'Batman faces off against the Joker, a criminal mastermind who threatens to plunge Gotham into chaos.', 'vector': '\\n/)j &*,\\'l$rR(\\x1d;ѥZ%, (*]-|\\x1f\\x03k+v(+$\\x1d)֭(6#բ0E+0() =&$9Y+*}\\x12+m(\\x1c(\\x11$I!V.()\"\\'l)J),\\x12*O Р,V\\'X$Ϥ&\\'\\x1a,\\x11\\')\\'N*ʠ1\\'\\x1c-::DJ)%\\'}(4\\x1fg\\'+S\\x08\\\\\\x07-\\x1f\\x10 !,*L)0**d),\\n--G!\\x15O)[Ȫ\\x00u\\x05\\x11T,fS%8)0!ps0e\\x07,\\x0c\\'7\\x0e\\x19g\\x08 g%\\x04\"(l*#R-c_)G)*\\x1adĭM\\x15̨߬#y ($쬮#9~)S\\x7f\\x0b#*\\x1fr\\x1fǡ5T\"+\\x19\"4\\'\\x17),\\x7fQ*d)\\t/T\\n\\x1c#ҭ¬\\r%\\x19s)릶(j*\\x00&@q9#\\x1dʬ*Y\\x1f8,(,*+\\x14)H&Eȡ\\x01\\n\\x17*#!\\t*l&(&((0gJ+\\x02)x\\x1c*\\x0f+Z.<(\\r(a$\\n( r`\\x1d}\\x11]1\\')N*\\x14%u+d(/\\x18\\x1fz\\x07ު\\r)!דɧ$+?(\\x10.\\x1f\\x12*\\x07*\\x05+%\\x00T˪]!\\x03$s+;Il*x%t% ,Z\\x06\\x0f=\\x18ȮC >\\',*ߣ3,\\x1a/*R$(t,ǩ\\\\$*~(;+2K& \\x1c\"uT\\x1d\\\\\\x11)Pfz$M)\\x15,\\x0b*b\\x1e', 'rating': '9'},\n", + " Document {'id': '7', 'payload': None, 'title': 'Gladiator', 'genre': 'action', 'description': 'A betrayed Roman general seeks revenge against the corrupt emperor who murdered his family.', 'vector': '.+\\x93;m^)\\x7f*\\x19\\x1a*3#\\x01\\x02.~$\\x02+$\\x1f|)e+X\\x04\\x04(C7,p\\x05\\x05M|+%.\\'*.50\\'~+(*$\\'t&\\x1a\\x11.\\x01\\x12\\'Ц+ (,\\x13\\'\\x1aWX\\';? \\x1c/Z+x\\x1d\\x1c\\x1a)\\x0f%jqT\\x14s*7\\x1e[*Fo,\\x0c0*hW*̠]-++h(o\\x1ebh)*\\x16\\x003\\x1clb!)*\\x0b\\x1dy.\\x1f@+,vV.,\\x197&(%[\\x1dQ(&\\'-(I#E!0$ЬU.X(-t(\\x1c&z$jED\\x1c(,$;\\x1cP*))\"),\\',(`^a$M5,\\x13\\x18a|(W9Ԯ\\x1e#i$\"\\x9d\\x08ޙ*\\x03!sì.l\\x00S\\x1f\\n$*\\x07!2k7\\'\\x1a$],V(ɪP0%##\\x1fj&\"z\\x0f).((,\\x0f,P#\\x01(G\\x1c./,\\x0bҡ$,\\'j,-˦)\\x08&h\\')&.\\x08.k(.v$̦\\'T\\x10$-Q(:r!(\\x07%O\\x1e\\'!,f-)*A\\x0f)B%t*}\\x0f,\\n.\\x1b-c\\x0c( .})\\x00\\x0c9&7\\x0f֬\\x1b\\x1f,!+\\r,\\x16%\\x1e\\\\\\x1cX\\x16\\x0f\\'ĥ\\x00-G\"\\x1e٥V, *̠!U,|+X\\',&m\\x02\\x01*,ʤQ\\x1d%,l\\x1a\\x15o*\\x12.(w#e\\x14', 'rating': '8'},\n", + " Document {'id': '8', 'payload': None, 'title': 'Inception', 'genre': 'action', 'description': 'A thief who enters dreams to steal secrets faces his toughest mission yet, with reality itself at stake.', 'vector': '**6$\\x18!\"&T+\\x1d١.$\"\\x0e3(%\\x01&*1\\x02**d\\')Щ\\uf805\\x1a-1*%+7էC \\x1c+ܡߡ\\x0f$\\x07**$~\\x1d(+ܫw($)[-\\x17\\x04+\\x0f(G2X-\\x1a+ףsp$%,hIf\\'\\x1cOg!-LA\\x10\\x0b#;\\n\\x03XR&,N-+Jx\\x1f;\\x00ܠ*!?#d(a&iAw\\x15Q.|\\'4*_8,a\\x14/Q8\\x0c*E9\\x1dW)m\\x1b\\x1d,N*&*-K*Wjh2\\'\\x11\\x15g&(\\x10)(\\x1aR\\x16\\x14\\x11),\\x00)9-\\x19*\\x17\\x19#-˨4\\x1aK\\x07ΩݦO&\\x02,\\x1f\\x0b7\\x05x(!\"-g%q\\x03@)\\':\\x07$\\x1d\\x00j-\\x16x\\'r,\\t,7\"\\x1e}*%)/\\r\\n*\\x06q #((\\n(N,o\\x16NE\\x1f߭M%aO-I[\\'y\\x11ħh\\\\\\\\,\\x1c,,\\x00-h)F)\\x03$\\x1b&s\\x1b()@+n%R\")(%`!.\\r)6-\\x11B&.!A((\\x13\\x1c,%J>*N(.ɫ\\x1b\\x08t$S\\x00{2+n*\\x02&W1K,*p\\'g)\\x19\\x18Ț)#-\"~\\'m# X0[&ɪd(\\x0e%6+)D&\\x13,\\x1d(,(\\x7f䧨,ʫ+\\x1eS\\'?\\x18\\x04-,m$M#4(^[\\x1c+\\x05\"\\x15\\x01', 'rating': '9'},\n", + " Document {'id': '9', 'payload': None, 'title': 'The Avengers', 'genre': 'action', 'description': \"Earth's mightiest heroes come together to stop an alien invasion that threatens the entire planet.\", 'vector': '&\\')\\x1b-0¨ȧ,=)\\x1c㞌)O\\x18\\'yu/4(-g+P!c\\x12\\x1dԧt$1\\x03*| m*\\x0fX&Ȭ[\\x1e\\x04\\x1cŬaC#p\\'Ϣ߫,w**\\x14ᠸ\\x1f$˥\\x1ff\\x1b*/\\x0c)\"\\x15<\\x14zc+,9%\\',[]&a\\x16@)4+$`-C+*ӣҪ&: w\\x195F&p \\x7f\\x1d>*0x*B\\x1d`**,)0\\x04\\t*$\\x07)|,x\\x12\\x00G\\x1b\\x1f(\\x1e&l\\x04-)C#_=)\\x0e(\\x1e$i(6\"C4\\x7f+\\x11(\\x03#!,Ξ\\x13J,Q-\"$&?(h+T,S\\x0c,j$d&+XT!f!\\x1b\\'\\x0c\\x1b%\\x18*\\x05+X)\\x1cΰ-%@\\x1f{--04\\t&$(\"X$w)\\x1cy zZ)y$)\\x02K\\x00\\x00˨\\x04\\x1a)\\x00/+)kr*\\x0f,\\x14?,:/\"M&`-(#\\ue840,#((@(\\x160\\x00-e!}C>\"()\\'\\x05+\\',\\x1d,\\x04w),m(a=+\\x1a-*\\'X,(\\x16ק!a,\\x18*+៥ڨs)\\x15?u\\x13$l\\x00~\\x1c,HX)(', 'rating': '8'}]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "res.docs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Does work with pure redis client.\n", + "client.memory_usage('rvl:37f6b0ea780c4a84a156f35f54a31c7e')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 793137da8fd4d632d04ead75ae1786fa6ed6b104 Mon Sep 17 00:00:00 2001 From: Robert Shelton Date: Fri, 27 Sep 2024 16:16:16 -0400 Subject: [PATCH 2/3] update llama --- python-recipes/RAG/03_llamaindex.ipynb | 263 ++++++++++++++++++++++--- 1 file changed, 235 insertions(+), 28 deletions(-) diff --git a/python-recipes/RAG/03_llamaindex.ipynb b/python-recipes/RAG/03_llamaindex.ipynb index b41a300..7d08e4b 100644 --- a/python-recipes/RAG/03_llamaindex.ipynb +++ b/python-recipes/RAG/03_llamaindex.ipynb @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -41,22 +41,7 @@ "id": "UQezgPCG1vml", "outputId": "97b9bc03-da1b-439a-c37b-be6fdb58ab21" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cloning into 'temp_repo'...\n", - "remote: Enumerating objects: 138, done.\u001b[K\n", - "remote: Counting objects: 100% (138/138), done.\u001b[K\n", - "remote: Compressing objects: 100% (98/98), done.\u001b[K\n", - "remote: Total 138 (delta 68), reused 91 (delta 35), pack-reused 0\u001b[K\n", - "Receiving objects: 100% (138/138), 7.19 MiB | 4.45 MiB/s, done.\n", - "Resolving deltas: 100% (68/68), done.\n", - "mv: rename temp_repo/resources to ./resources: Directory not empty\n" - ] - } - ], + "outputs": [], "source": [ "# NBVAL_SKIP\n", "!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\n", @@ -200,7 +185,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Sample doc Doc ID: 67e07154-6ea0-4822-8957-ac1d212fc9ee\n", + "Sample doc Doc ID: c013353e-dae7-4d17-befd-9e784c8acf79\n", "Text: UNITED STATES SECURITIES AND EXCHANGE COMMISSION Washington,\n", "D.C. 20549 FORM 10-K (Mark One) ☒ ANNUAL REPORT PURSUANT T O SECTION\n", "13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 For the fiscal year\n", @@ -245,13 +230,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from llama_index.core import StorageContext\n", "\n", - "vector_store = RedisVectorStore(redis_url=REDIS_URL, index_name=\"llama\", overwrite=True)\n", + "vector_store = RedisVectorStore(redis_url=REDIS_URL, overwrite=True)\n", "\n", "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", "\n", @@ -267,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -285,14 +270,14 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Node ID: b561dd17-5545-4d3a-bc4f-18cb39c7c01e\n", + "Node ID: d2e6cd9c-0716-49d8-8563-407a00d05445\n", "Text: Table of Contents FISCAL 2023 NIKE BRAND REVENUE HIGHLIGHTS The\n", "following tables present NIKE Brand revenues disaggregated by\n", "reportable operating segment, distribution channel and major product\n", @@ -301,14 +286,14 @@ "fiscal 2022 on...\n", "Score: 0.900\n", "\n", - "Node ID: 0415f059-9258-426b-8b21-34b287b3c21b\n", + "Node ID: 28542d3b-b345-4e9e-b675-f62361ec85d9\n", "Text: Table of Contents NORTH AMERICA (Dollars in millions) FISCAL\n", "2023FISCAL 2022 % CHANGE% CHANGE EXCLUDING CURRENCY CHANGESFISCAL 2021\n", "% CHANGE% CHANGE EXCLUDING CURRENCY CHANGES Revenues by: Footwear $\n", "14,897 $ 12,228 22 % 22 %$ 11,644 5 % 5 % Apparel 5,947 5,492 8 % 9 %\n", "5,028 9 % 9 % Equipment 764 633 21 % 21 % 507 25 % 25 % TOTAL REVENUES\n", "$ 21,6...\n", - "Score: 0.886\n", + "Score: 0.885\n", "\n" ] } @@ -329,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -338,7 +323,7 @@ "\"NIKE's revenue in fiscal 23 was $51.2 billion.\"" ] }, - "execution_count": 13, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -348,6 +333,228 @@ "response.response" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use a custom index schema\n", + "\n", + "In most use cases, you need the ability to customize the underling index configuration\n", + "and specification. For example, this is handy in order to define specific metadata filters you wish to enable.\n", + "\n", + "With Redis, this is as simple as defining an index schema object\n", + "(from file or dict) and passing it through to the vector store client wrapper." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from redisvl.schema import IndexSchema\n", + "\n", + "\n", + "custom_schema = IndexSchema.from_dict(\n", + " {\n", + " # customize basic index specs\n", + " \"index\": {\n", + " \"name\": \"custom_index\",\n", + " \"prefix\": \"docs\",\n", + " \"key_separator\": \":\",\n", + " },\n", + " # customize fields that are indexed\n", + " \"fields\": [\n", + " # required fields for llamaindex\n", + " {\"type\": \"tag\", \"name\": \"id\"},\n", + " {\"type\": \"tag\", \"name\": \"doc_id\"},\n", + " {\"type\": \"text\", \"name\": \"text\"},\n", + " # custom metadata fields\n", + " {\"type\": \"numeric\", \"name\": \"updated_at\"},\n", + " {\"type\": \"tag\", \"name\": \"file_name\"},\n", + " # custom vector field definition for cohere embeddings\n", + " {\n", + " \"type\": \"vector\",\n", + " \"name\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 1536,\n", + " \"algorithm\": \"hnsw\",\n", + " \"distance_metric\": \"cosine\",\n", + " },\n", + " },\n", + " ],\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "IndexInfo(name='custom_index', prefix='docs', key_separator=':', storage_type=)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "custom_schema.index" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': TagField(name='id', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'doc_id': TagField(name='doc_id', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'text': TextField(name='text', type='text', path=None, attrs=TextFieldAttributes(sortable=False, weight=1, no_stem=False, withsuffixtrie=False, phonetic_matcher=None)),\n", + " 'updated_at': NumericField(name='updated_at', type='numeric', path=None, attrs=NumericFieldAttributes(sortable=False)),\n", + " 'file_name': TagField(name='file_name', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'vector': HNSWVectorField(name='vector', type='vector', path=None, attrs=HNSWVectorFieldAttributes(dims=1536, algorithm=, datatype=, distance_metric=, initial_cap=None, m=16, ef_construction=200, ef_runtime=10, epsilon=0.01))}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "custom_schema.fields" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# from datetime import datetime\n", + "\n", + "\n", + "# def date_to_timestamp(date_string: str) -> int:\n", + "# date_format: str = \"%Y-%m-%d\"\n", + "# return int(datetime.strptime(date_string, date_format).timestamp())\n", + "\n", + "\n", + "# # iterate through documents and add new field\n", + "# for document in docs:\n", + "# document.metadata[\"updated_at\"] = date_to_timestamp(\n", + "# document.metadata[\"last_modified_date\"]\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "vector_store = RedisVectorStore(\n", + " schema=custom_schema, # provide customized schema\n", + " redis_url=REDIS_URL,\n", + " overwrite=True,\n", + ")\n", + "\n", + "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "# build and load index from documents and storage context\n", + "index = VectorStoreIndex.from_documents(\n", + " docs, storage_context=storage_context\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query the vector store and filter on metadata\n", + "Now that we have additional metadata indexed in Redis, let's try some queries which add in filters. As an example, we'll do a search for chunks with the word \"audit\" from an exact file \"amzn-10k-2023.pdf\". " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core.vector_stores import (\n", + " MetadataFilters,\n", + " MetadataFilter,\n", + " ExactMatchFilter,\n", + ")\n", + "\n", + "retriever = index.as_retriever(\n", + " similarity_top_k=3,\n", + " filters=MetadataFilters(\n", + " filters=[\n", + " ExactMatchFilter(key=\"file_name\", value=\"amzn-10k-2023.pdf\"),\n", + " MetadataFilter(\n", + " key=\"text\",\n", + " value=\"audit\",\n", + " operator=\"text_match\",\n", + " ),\n", + " ],\n", + " condition=\"and\",\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Node ID: cd0c5d8f-e3b1-4cbb-aa6a-5960003cdb2d\n", + "Text: Table of Contents valuation. In the ordinary course of our\n", + "business, there are many transactions and calculations for which the\n", + "ultimate tax determination is uncertain. Significant judgment is\n", + "required in evaluating and estimating our tax expense, assets, and\n", + "liabilities. We are also subject to tax controversies in various\n", + "jurisdictions that can...\n", + "Score: 0.746\n", + "\n", + "Node ID: 6745f668-4c7a-43bf-a9c3-9b04e1a497f8\n", + "Text: Table of Contents Included in other income (expense), net in\n", + "2021 and 2022 is a marketable equity securities valuation gain (loss)\n", + "of $11.8 billion and $(12.7) billion from our equity investment in\n", + "Rivian Automotive, Inc. (“Rivian”). Our investment in Rivian’s\n", + "preferred stock was accounted for at cost, with adjustments for\n", + "observable changes in ...\n", + "Score: 0.740\n", + "\n", + "Node ID: 717666fe-fea5-488b-999c-84e6d8b9a0db\n", + "Text: Exhibit 31.1 CERTIFICATIONS I, Andrew R. Jassy, certify that: 1.\n", + "I have reviewed this Form 10-K of Amazon.com, Inc.; 2. Based on my\n", + "knowledge, this report does not contain any untrue statement of a\n", + "material fact or omit to state a material fact necessary to make the\n", + "statements made, in light of the circumstances under which such\n", + "statements were ...\n", + "Score: 0.732\n", + "\n" + ] + } + ], + "source": [ + "result_nodes = retriever.retrieve(\"What did the author learn?\")\n", + "\n", + "for node in result_nodes:\n", + " print(node)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -376,7 +583,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.11.9" }, "widgets": { "application/vnd.jupyter.widget-state+json": { From 21104d8a1b6d8f78a24b435a41cef11329a57a68 Mon Sep 17 00:00:00 2001 From: Robert Shelton Date: Fri, 27 Sep 2024 16:19:06 -0400 Subject: [PATCH 3/3] remove unrelated file --- .../vector-search/02_float16_support.ipynb | 637 ------------------ 1 file changed, 637 deletions(-) delete mode 100644 python-recipes/vector-search/02_float16_support.ipynb diff --git a/python-recipes/vector-search/02_float16_support.ipynb b/python-recipes/vector-search/02_float16_support.ipynb deleted file mode 100644 index bddd147..0000000 --- a/python-recipes/vector-search/02_float16_support.ipynb +++ /dev/null @@ -1,637 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'5.0.8'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import redis\n", - "redis.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![Redis](https://redis.io/wp-content/uploads/2024/04/Logotype.svg?auto=webp&quality=85,75&width=120)\n", - "# Using smaller vector types\n", - "\n", - "With the [Redis 7.4 release](https://redis.io/blog/announcing-redis-community-edition-and-redis-stack-74/) there is now support for bfloat16 and float16 data types in the vector store. Using this type is exactly the same as using float32 or other data types but will require a conversion if seeking to replace previously stored objects.\n", - "\n", - "This tutorial will walk through repopulating and index as such.\n", - "\n", - "## Packages" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "^C\n", - "\u001b[31mERROR: Operation cancelled by user\u001b[0m\u001b[31m\n", - "\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "# NBVAL_SKIP\n", - "%pip install -q redis redisvl numpy sentence-transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install Redis Stack\n", - "\n", - "Later in this tutorial, Redis will be used to store, index, and query vector\n", - "embeddings created from PDF document chunks. **We need to make sure we have a Redis\n", - "instance available.\n", - "\n", - "#### For Colab\n", - "Use the shell script below to download, extract, and install [Redis Stack](https://redis.io/docs/getting-started/install-stack/) directly from the Redis package archive." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# NBVAL_SKIP\n", - "%%sh\n", - "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", - "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", - "sudo apt-get update > /dev/null 2>&1\n", - "sudo apt-get install redis-stack-server > /dev/null 2>&1\n", - "redis-stack-server --daemonize yes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### For Alternative Environments\n", - "There are many ways to get the necessary redis-stack instance running\n", - "1. On cloud, deploy a [FREE instance of Redis in the cloud](https://redis.com/try-free/). Or, if you have your\n", - "own version of Redis Enterprise running, that works too!\n", - "2. Per OS, [see the docs](https://redis.io/docs/latest/operate/oss_and_stack/install/install-stack/)\n", - "3. With docker: `docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest`\n", - "\n", - "##### if running local version make sure to double check it is updated >=7.4\n", - "\n", - "### Define the Redis Connection URL\n", - "\n", - "By default this notebook connects to the local instance of Redis Stack. **If you have your own Redis Enterprise instance** - replace REDIS_PASSWORD, REDIS_HOST and REDIS_PORT values with your own." - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Replace values below with your own if using Redis Cloud instance\n", - "REDIS_HOST = os.getenv(\"REDIS_HOST\", \"localhost\") # ex: \"redis-18374.c253.us-central1-1.gce.cloud.redislabs.com\"\n", - "REDIS_PORT = os.getenv(\"REDIS_PORT\", \"6379\") # ex: 18374\n", - "REDIS_PASSWORD = os.getenv(\"REDIS_PASSWORD\", \"\") # ex: \"1TNxTEdYRDgIDKM2gDfasupCADXXXX\"\n", - "\n", - "# If SSL is enabled on the endpoint, use rediss:// as the URL prefix\n", - "REDIS_URL = f\"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# from redis import Redis\n", - "import redis\n", - "\n", - "client = redis.Redis(\n", - " host='redis-11118.c309.us-east-2-1.ec2.redns.redis-cloud.com',\n", - " port=11118,\n", - " password='qt1Zl62JL07F7RTGac40XCiEhksh4xjD')\n", - "# client = Redis.from_url(REDIS_URL)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "client.ping()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Define index and load data\n", - "\n", - "We will be loading a schema of movie information with a 32bit float." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "with open(\"resources/movies.json\", 'r') as file:\n", - " movies = json.load(file)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "07:32:48 redisvl.index.index INFO Index already exists, overwriting.\n" - ] - } - ], - "source": [ - "from redisvl.schema import IndexSchema\n", - "from redisvl.index import SearchIndex\n", - "\n", - "index_name = \"movies32\"\n", - "\n", - "schema = IndexSchema.from_dict({\n", - " \"index\": {\n", - " \"name\": index_name,\n", - " },\n", - " \"fields\": [\n", - " {\n", - " \"name\": \"title\",\n", - " \"type\": \"text\",\n", - " },\n", - " {\n", - " \"name\": \"description\",\n", - " \"type\": \"text\",\n", - " },\n", - " {\n", - " \"name\": \"genre\",\n", - " \"type\": \"tag\",\n", - " \"attrs\": {\n", - " \"sortable\": True\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"rating\",\n", - " \"type\": \"numeric\",\n", - " \"attrs\": {\n", - " \"sortable\": True\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"vector\",\n", - " \"type\": \"vector\",\n", - " \"attrs\": {\n", - " \"dims\": 384,\n", - " \"distance_metric\": \"cosine\",\n", - " \"algorithm\": \"hnsw\",\n", - " \"datatype\": \"float32\"\n", - " }\n", - " }\n", - " ]\n", - "})\n", - "\n", - "\n", - "index = SearchIndex(schema, client)\n", - "index.create(overwrite=True, drop=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embed movie description vectors" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", - " warnings.warn(\n", - "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "import numpy as np\n", - "from sentence_transformers import SentenceTransformer\n", - "\n", - "# load model for embedding our movie descriptions\n", - "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n", - "\n", - "def embed_text(model, text):\n", - " return np.array(model.encode(text)).astype(np.float32).tobytes()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# Note: convert embedding array to bytes for storage in Redis Hash data type\n", - "movie_data32 = [\n", - " {\n", - " **movie,\n", - " \"vector\": embed_text(model, movie[\"description\"])\n", - " } for movie in movies\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Populate the index" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['rvl:37b42d13adaa4a4ea32db5e4d062126b',\n", - " 'rvl:9e3dfc7c67b744bb8199d364c60f8d89',\n", - " 'rvl:8f7758c7763842f78490e3c43aab1148',\n", - " 'rvl:c961854993a04d089da2e026033b80b3',\n", - " 'rvl:7f2699e7dbf3474283d34bf04b465f00',\n", - " 'rvl:cc56f14da1f8431c90a318167ce7e506',\n", - " 'rvl:5c0097099939492f87b2b56890bc7dfe',\n", - " 'rvl:6c215979299c424ebff07a4b85a60f43',\n", - " 'rvl:896cd78dae42483db66019f2b62a70e4',\n", - " 'rvl:b92104d038c04ab3beab80139f75e7d1',\n", - " 'rvl:17b08c1a64b743ecab2e5acaa4a777af',\n", - " 'rvl:d7e8d94d158e4003ad443ec5ce517f53',\n", - " 'rvl:a448ffc20ac8474ab776df5e54ae26a9',\n", - " 'rvl:87e2cd98e776411cb7f552a29a808a2a',\n", - " 'rvl:750ccddcf5604b12b33fb81e78042db0',\n", - " 'rvl:403fcaf607b04e5a9b7561a86082cb26',\n", - " 'rvl:12e4333184554daca7cbbf44e77ccb18',\n", - " 'rvl:5be2ed277c1c4f3b8afb929446afda9e',\n", - " 'rvl:6ccdd8c9eb1f433fa92c85f16d4351ff',\n", - " 'rvl:7ba0ee30d58f45e7bc1d5d21fbe9a45d']" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "keys = index.load(movie_data32)\n", - "keys" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "client.memory_usage('rvl:37f6b0ea780c4a84a156f35f54a31c7e')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Now let's convert to fp16\n", - "\n", - "This is as simple as you think it is using numpy." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "movie_data16 = []\n", - "for movie in movie_data32:\n", - " vect16 = np.frombuffer(movie[\"vector\"], dtype=np.float32).astype(np.float16)\n", - " movie[\"vector\"] = vect16.tobytes()\n", - " movie_data16.append(movie)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'title': 'Explosive Pursuit',\n", - " 'genre': 'action',\n", - " 'rating': 7,\n", - " 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.',\n", - " 'vector': b'\\xe3+S\\x18\\xbd\\x1d\\xf6\\xabs\\x9eQ*\\xfd)\\xac$8\\xb0{.\\x80+o\\xa8H\\xaa3&\\x0f+\\xb2\\xa0\\x9f \\xe5(O$n!\\xfe\\xa4\\x18\\xae\\x92*\\xbb\\xa7\\xba\\xb0)\\x88\\xfc%\\x17\\x93E\\xadg\\xa7+)\\x97\\x9b:*),\\x12\\xa5\\x83\\xaeY\\xaa\\xa7\\xa4G*1\"a\\'o\\xac\\xc3$\\x1d-\\x9b\\xa9\\xee\\xad\\xb9\\x1f\\xd8+f,0\\xae\\x82\\xac\\xe9\\x1dg\\xa8J\\xa6\\xe0\\xa5\\xa9\\x9bT&T*U)\\xaa\\xaa\\xb0\\xa4\\xc8(`\"N(d-\\xc7\\xad\\xe6\\xac\\xc4(3)\\xc2\\'H$\\x94\\x9f\\xda\\xaau\\x9f\\n\\xaa\\xcb\\xa8\\x93\\x9f\\xc6\\x1bz\"b\\xa1u-h\\xa1p\\xad\\xd7(n.\\x81)\\xf0\\xa4Z*\\x06)P\\xaa\\xf3\\x9c\\xbc\\x17\\xe8(\\xf6&\\xb3\\x98\\x83\\x9b\\xe7!\\x1d\\x85\\x9a#2!\\xc1 o\\xa9\\xc7$u*)-\\xf0&\\x9a p\\x96\\xe5\\xa6*(\\xad\\'\\x97\\xa2m\\xa1\\xe7\\nb/\\xb8*\\xb2\\xa6\\x99*\\xf7\\xaa\\x8d$\\x9f \\xce\\x9b|\\xa9\\x80#E\\x9c\\xef$\\xb0-\\x00\\x808\\x9d0\\xab[\\x9a\\x81\\x9c\\xcc\\x9d\\xce\\x9d[+h%C\\xadc.\\x07\\xb1z\\xa9\\xbb\\xa7\\xb5+\\x88#\\xe7\\xa8X$3\\xa54$\\xcc(%\\xa7\\xf3\\xa2s\\xa8\\x1e\\xa7\\xaa\\x1a\\xe2\\xab`\\xacT\\xa9\\xa4/\\x86\\x18q\\xa8\\xa7-\\xa9\\xa8\\x1c,\\xd1!\\x0f%h\\xacH\\xab\\xe8&\\xb4.\\xe5\\xac\\x0c\\xa9)\\x8e\\x9e\\xa4\\xf6\\xa91\\xaa\\xa2\\xb0D\\xa9J\\xac\\xa5&\\x19.x\\xaa\\x84\\xa3f\\xa5-\\xab\\x9c\\x1b\\x0b\\x1e\\x18\\xaa\\xf1\\x17\\xa7/Q!\\xe0(\\xb8\\xaao(7(\\xd4\\xad\\xb5&\\xcf\\x1ez).,\\x10){\\'\\x1f+B\\xb0\\xd9)\\xbb\\x1a\\x15#\\xbd%\\xdc\\xa9\\x8c\\xa4\\xe0\\xa9\\x19%*\\xa1\\xff\\x9d\\x94(\\xa9,\\xec\\xa7\\x90\\xae$\\x1c\\xa8+L#\\x95\\xaap\\xab\\x9e)7*\\x00\\x00\\xd2%}\\xaa8\\x1e(\\xa8\\x94\\xac\\'*\\xdf\\xa1\\xa2\\xa7\\xbc\\xack\\xa8\\xac\\xafA)6.\\xd7(\\xc5/v\\xa7\\xc1\\x98\\xdc!J\\xa8\\xb8(\\xf5(Z\\xaeh\\xack\\x9f\\xcb\\xa0\\x98\\xa0\\x1c,$\\xab\\x8e\\x9e\\xcb\\xa2\\x0c+\\xee\\xa7\\x1d\\x9c\\xca\\xa8\\xe8\\xaa\\xd7+\\xe2,K\\xad\\xcb\\xa1w\\xa2\\xc0\\xab\\x04-\\xb5$\\xc4\\x91\\xee\\xa9D!\\xa7\\x9d^\\xa8\\xa8\\xac\\xa6*\\x8c&:\\xa9@\"\\xc0%\\xeb\\xab\\x0c(\\xde\\xaa\\x1f {\\xa8<\\xa7A&\\xe0,7\\xad+/y\"\\x98)\\xb2\\xa9\\x97\\xa7;%\\x16\\xb0\\x93\\xa9T,N\\xa8n\\xa6o\\x9d\\xf8\\xa6)*\\xe8-i+s,%-U\\x9ej\\xa0\\xe2$s\\xacY+\\xf8\\x1d\\xf9\\'\\x92$b\\x1c\\x1d-\\xb1#\\x84\\x9f?\\xa0\\xed\\xa5\\x00\\x80\\x9c\\xaeW+?\\xad)\\'\\x81+\\xff,B\\xab\\xa5\\xaf?\\xa0W#\\xbc+\\x0f\\'\\xf5(\"&R #\\xa3h,4\\xa5i\\x96|.k\\xa9s+\\xf9\\xac\\xfc*\\x19!m\\x1b\\x04\\xb0\\\\\\xa4;*U.\\x8e\\xa3\\xd7\\xa9\\xed\"\\x8b\\x1es-\\x19\\xa8J,\\x9a\\xab\\x11+N\\x98@\\'&\\xa7\\x1b)\\\\\\xab\\x0b w\\xaa\\x88\\xa5P\\xaa\\xf8+\\xd7\\xaa\\x89\\x9f\\xc2)\\xa2\\xa2\\t$m*/ \\xe2\\xab\\xbd#?\\xad\\x03\\xa99(\\xa1\\xa0L\\'\\xb9,'}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "movie_data16[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Create new index" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "ename": "ValidationError", - "evalue": "1 validation error for IndexSchema\n__root__ -> attrs -> datatype\n value is not a valid enumeration member; permitted: 'FLOAT32', 'FLOAT64' (type=type_error.enum; enum_values=[, ])", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[12], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mredisvl\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mindex\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SearchIndex\n\u001b[1;32m 4\u001b[0m index_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmovies16\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 6\u001b[0m schema \u001b[38;5;241m=\u001b[39m \u001b[43mIndexSchema\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mindex\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfields\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtitle\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 15\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdescription\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 18\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 19\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 20\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgenre\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtag\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 22\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msortable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 24\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 25\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 26\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrating\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 28\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnumeric\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 29\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 30\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msortable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 31\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 32\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 33\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 34\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvector\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 35\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvector\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattrs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdims\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m384\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdistance_metric\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcosine\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43malgorithm\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhnsw\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdatatype\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfloat16\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\n\u001b[1;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 44\u001b[0m \u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m index \u001b[38;5;241m=\u001b[39m SearchIndex(schema, client)\n\u001b[1;32m 48\u001b[0m index\u001b[38;5;241m.\u001b[39mcreate(overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", - "File \u001b[0;32m~/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/schema/schema.py:263\u001b[0m, in \u001b[0;36mIndexSchema.from_dict\u001b[0;34m(cls, data)\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[1;32m 228\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfrom_dict\u001b[39m(\u001b[38;5;28mcls\u001b[39m, data: Dict[\u001b[38;5;28mstr\u001b[39m, Any]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIndexSchema\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 229\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Create an IndexSchema from a dictionary.\u001b[39;00m\n\u001b[1;32m 230\u001b[0m \n\u001b[1;32m 231\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124;03m })\u001b[39;00m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.pyenv/versions/3.11.9/lib/python3.11/site-packages/pydantic/v1/main.py:341\u001b[0m, in \u001b[0;36mBaseModel.__init__\u001b[0;34m(__pydantic_self__, **data)\u001b[0m\n\u001b[1;32m 339\u001b[0m values, fields_set, validation_error \u001b[38;5;241m=\u001b[39m validate_model(__pydantic_self__\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, data)\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validation_error:\n\u001b[0;32m--> 341\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m validation_error\n\u001b[1;32m 342\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 343\u001b[0m object_setattr(__pydantic_self__, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__dict__\u001b[39m\u001b[38;5;124m'\u001b[39m, values)\n", - "\u001b[0;31mValidationError\u001b[0m: 1 validation error for IndexSchema\n__root__ -> attrs -> datatype\n value is not a valid enumeration member; permitted: 'FLOAT32', 'FLOAT64' (type=type_error.enum; enum_values=[, ])" - ] - } - ], - "source": [ - "from redisvl.schema import IndexSchema\n", - "from redisvl.index import SearchIndex\n", - "\n", - "index_name = \"movies16\"\n", - "\n", - "# this won't work till we merge Justin's stuff\n", - "schema = IndexSchema.from_dict({\n", - " \"index\": {\n", - " \"name\": index_name,\n", - " },\n", - " \"fields\": [\n", - " {\n", - " \"name\": \"title\",\n", - " \"type\": \"text\",\n", - " },\n", - " {\n", - " \"name\": \"description\",\n", - " \"type\": \"text\",\n", - " },\n", - " {\n", - " \"name\": \"genre\",\n", - " \"type\": \"tag\",\n", - " \"attrs\": {\n", - " \"sortable\": True\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"rating\",\n", - " \"type\": \"numeric\",\n", - " \"attrs\": {\n", - " \"sortable\": True\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"vector\",\n", - " \"type\": \"vector\",\n", - " \"attrs\": {\n", - " \"dims\": 384,\n", - " \"distance_metric\": \"cosine\",\n", - " \"algorithm\": \"hnsw\",\n", - " \"datatype\": \"float16\"\n", - " }\n", - " }\n", - " ]\n", - "})\n", - "\n", - "\n", - "index = SearchIndex(schema, client)\n", - "index.create(overwrite=True, drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "from redis.commands.search.field import VectorField, TagField, NumericField, TextField\n", - "from redis.commands.search.indexDefinition import IndexDefinition, IndexType\n", - "\n", - "index_name = \"movies\"\n", - "\n", - "schema = (\n", - " VectorField(\n", - " \"vector\",\n", - " \"HNSW\",\n", - " {\n", - " \"TYPE\": \"FLOAT16\",\n", - " \"DIM\": 384,\n", - " \"DISTANCE_METRIC\": \"COSINE\"\n", - " }\n", - " ),\n", - " NumericField(\"rating\"),\n", - " TagField(\"genre\"),\n", - " TextField(\"title\"),\n", - " TextField(\"description\")\n", - ")\n", - "\n", - "try:\n", - " client.ft(index_name).info()\n", - " print(\"Index exists!\")\n", - "except:\n", - " # index Definition\n", - " definition = IndexDefinition(index_type=IndexType.HASH)\n", - "\n", - " # create Index\n", - " client.ft(index_name).create_index(fields=schema, definition=definition)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def load_docs(client: redis.Redis, data: list[dict]):\n", - " for i, d in enumerate(data):\n", - " client.hset(\n", - " i,\n", - " mapping = d\n", - " )\n", - "\n", - "def print_results(res):\n", - " docs = [(doc.title, doc.genre, doc.rating) for doc in res.docs]\n", - " print(f\"Top {len(docs)} movies: \", docs)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "load_docs(client, movie_data16)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Result{20 total, docs: [Document {'id': '0', 'payload': None, 'title': 'Explosive Pursuit', 'genre': 'action', 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.', 'vector': '+S\\x18\\x1dsQ*)$8{.+oH3&\\x0f+ (O$n!\\x18*)%\\x17Eg+):*),\\x12YG*1\"a\\'o$\\x1d-\\ueb79\\x1f+f,0\\x1dgJ३T&T*U)(`\"N(d-ǭ(3)\\'H$ڪu\\n˨\\x1bz\"bu-hp(n.)Z*\\x06)P\\x17(&!\\x1d#2! o$u*)-& p*(\\'m\\nb/**$ Λ|#E$-\\x0080[̝Ν[+h%Cc.\\x07z+#X$34$(%s\\x1e\\x1a`T/\\x18q-\\x1c,!\\x0f%hH&.\\x0c)1DJ&\\x19.xf-\\x1b\\x0b\\x1e\\x18\\x17/Q!(o(7(ԭ&\\x1ez).,\\x10){\\'\\x1f+B)\\x1a\\x15#%ܩ\\x19%*(,짐$\\x1c+L#p)7*\\x00\\x00%}8\\x1e(\\'*ߡkA)6.(/v!J((Zhkˠ\\x1c,$ˢ\\x0c+\\x1dʨ+,Kˡw\\x04-$đD!^*&:@\"%\\x0c(ު\\x1f {\\x07(.\\x1c@.\\x1e[)߬,\"5(\\x1f([\\x00|7\\x1e+19A-\\x16%\\x1f$\\x08,KX\\x1d*.m-ۢ寋\\x1f)\\x17v(*\"3%N)&\\x1f\\x1f\\x02\\x15\\x04\\x16,\\x9ey!Z+\\x12|\\x08$Uz%\\x1cRh\\x0c\\x18(#N\\x1dGJdÕX\\x0c\\x19蜗./\\x1b\"̘%)b+**.Ϫ\\x00 @,_-\\x042y+.1\\x1fS{)d\\x00\\x00o\\x1eɪ:\\x7f,\\x08w\"] \\x1c.W\\'!\\x00)Ѧ\\x06\\'\\x04-\\x1b\\x17.L %\\x1c,6$+M\\x1du\\x1f\\'e0#*(z)V,$\\x02ܬ\"&p%\\r2\\x1f-D\\'t*9\\x03ά%˩%$\\x1aW)%,<\\x15,!%\\x1f!k\\x1e&\\x16*)\\x17\\x1e((u*(>\\x03[\\x1cc,%\\x003p(w*\\x16\\x1f4 =)+]\\x14\\x18\\x1a#\\x1e,G\\x1f!\\'ڦ\\',\\x1c\\x1f)\\t$j/)N)\\x16(\\x0co+t? \\x06.-ר+\\x18,\\x18ԫf*O)\\n kũ`*)\\x0f\\'m\\\\%B#\\x17)', 'rating': '8'}, Document {'id': '2', 'payload': None, 'title': 'Fast & Furious 9', 'genre': 'action', 'description': 'Dom and his crew face off against a high-tech enemy with advanced weapons and technology.', 'vector': '=\\x11-A\\x1d*+\\x01-g4);(m+\\x1c\\rT,\\x1e+b#\\x1bء(\\x18\\x00.q-uڥL(֦\\x08>}-\\x1d*O¬P \\x18\\x05)J\\x1a,\\x1b*!)\\x19&\\x14R,A#Wo&ʡA+\\r+),-\\x1dU#ߤ&&\\x1fᗬ/۩?*3˕~+Ȩ._)s&\\x02.\\x1c\\x11-,$%<+)\\x04\\'(,d$1g3*![%8\\t5)V1(LӨ|.-*\\x00\\x19\"Z+(* c-\\x19ǩ$A\\x1d\\x1c*ޣ\\t\\x16M\\'#E%,l\\x155\\x1f)-\\x00\\x15,\\x10*1-(ͥȫj\\x11I$\\x13\\'\"+\\x19%˨#\\'\\'\\'\\u2438-(**,*˦[t)r\\x02)H*U(먤%p\"$#$\\x1b$\\x0eb\\'AC)?\\x1aD\\x14dާ\\x00\\x008b& u*z\\x1f,+],)Bp5r,\\x0c\\x12,S\\x1c(1y\\',)Z*p(\\x19+\\x07$e 0*ө_\\x0b,Ĥ~7.vQ\\x1c\\x15&~) c,G,\\x1b!;*\\x07\\x1d\\x10(J-\\x1b,/.ά*ik) \\x1cp*k(\\x1e¨-)c/ɝ\\x12\\\\w{\\x1f\\x1c9\\x00dG\\x01\\x1f&۩h,A\\x1d$ -h\\'R+,\\x0bcԮ\\x08\\x1d).(o%-z&,-; \\x1c0\\x13*Μ,ҭ*H\\'v,K-\\x90\\x02\\x13\\x1d\\x04)$-Ir$B 9\\x15,R(', 'rating': '6'}, Document {'id': '3', 'payload': None, 'title': 'Black Widow', 'genre': 'action', 'description': 'Natasha Romanoff confronts her dark past and family ties as she battles a new enemy.', 'vector': '/~\\x17+%\\x04-Z(Δ\\x1f?\\x1c\\x16$6:B\\x14$\\x1c\\r\\x1ff\\x0b#)\\x06\\'(6\\x1c&\\x13),\"&$*-y)&)E*R\\x00u\\x16$.\\x1d\\x07\\x14V\\x1dXn)-.)\\x1d\\'\\'*]J\\x13)\\'%{,Y\\x07*+D˪\\x11..(%\\x15))\"M!(\\x17\\n!R+\\x02E\\x1f ^,\\'&Ց$\\x10**\\x0f1(ڨ)rڬ)+&\\x17.\\x01 /\\x18oK,J)[*\\x06%T\\x1f8\\x00N#%K\\x16/(!)r(}\\x1eV,x`)d*j(#/z\\x1d/0\\x13?ӧ\\n(6(}$*%\\'-$d*\\'(\\x06\\x14\\x1e\\x13`(JY4#v,4T!<\\x02\\x00*$\\x17-\\'i5*\\x14+\\x18\\x19ҥi&($.\\x1c t\\x1a!)\\x0f,6۬7\\'A*\\x04\\x1f\\x08(\\x02\\x05)|&\\x00-k\\x7f&,⡡X.P!\\x0b)!(2(#Ѫ.\\x1e\\x94\\x10\\x1a\\x06`,j0}\\x15oG\\x06\\x1b\\'*0*ˮ\\'$8\\x0b-?-m/u)0\\x0eXt*~,ί)\\x1f,/D\\x10%v\\x00zl\\x1f٩j-̦2\\x1b,]$\\x0b\\'$)F\\',\\x13/&}.E(gK\\x14L\\x00硇R\\'Ȭժ+\\x1bb\\x1a\\x1d*\\x05\"j)#Ϊ1.k-7we\\x13\\x05\\'g\\x15rΥ\\t,)%!Ҩy-%f\\x1fwg*P,hRw\"*+1<\\x18\\x1c\\x1a)m(8)\"\\x0c*%,h*U', 'rating': '7'}, Document {'id': '4', 'payload': None, 'title': 'John Wick', 'genre': 'action', 'description': 'A retired hitman seeks vengeance against those who wronged him, leaving a trail of destruction in his wake.', 'vector': '\\x9b).\\x19$\\x1b\\r#Mդk\\'\\x03H(p\\x1f%2$.l$@:-/\\x11QSѥ\\x03*%թţ\\x1cޭO$*H#%%h$ʩ\\x0f!C+\\x00*fa((h(kx-y\\x1ch*ĢQ()d%\\\\,U(j,)E$\\x0e|$%,Q!^-* #*K+)$%1\\x10,˧*k.؟l5#()$ѯ\\x15,\\x0eϘd(\\x1c,窺;,4(!k\\x1d\\x19,\\x1d\\x00(&& \\x00y,#\\n\\'w%\\x16&a)&*y\\x11/$\"?#,ެP\\x08.ǥ(\\x17g#(5xi/,U*!#a(u(\\x0c\\x16\\x19T$.h1!0%%w\\x01,[\">N\\'A(4\\x19+\\x03Y\\x1f.\"\\x1e\\n:\\x1d-<\\x1e)\\x00P)D*ڦ\\x15%&ʨե&\\'\\x05\\x01\\t&+$$p,=\\x00\\x18#\\')\\x03,\\x1e,,t4>1/\\x1f#\\x10QM\"=-\\x0b̟],2\\x0b#)H-9.& V#h\\x1e\\'$\\x02$+0\\x16+.\\x06)\\'\\'-\\'d*E\\x1eŬ \\x061\\x14R)ƩVW*\\x10XI&ͫ\\x04\"#\\'ҭ\\x17,\\x0f\\x1fB)#\\'!ӟ\\x00\\x04ꕔ%\\x12 +!]m*\\x19%o--s\\x1e$(Nάܪ\\x1cq\"(,N*&\\x1eB) \\x0e(y%p)+\\x10?A,(\\'3ǮW $wt)+\"\\x0c)t)yY(!Ƥ0\\x0536\"', 'rating': '8'}, Document {'id': '6', 'payload': None, 'title': 'The Dark Knight', 'genre': 'action', 'description': 'Batman faces off against the Joker, a criminal mastermind who threatens to plunge Gotham into chaos.', 'vector': '\\n/)j &*,\\'l$rR(\\x1d;ѥZ%, (*]-|\\x1f\\x03k+v(+$\\x1d)֭(6#բ0E+0() =&$9Y+*}\\x12+m(\\x1c(\\x11$I!V.()\"\\'l)J),\\x12*O Р,V\\'X$Ϥ&\\'\\x1a,\\x11\\')\\'N*ʠ1\\'\\x1c-::DJ)%\\'}(4\\x1fg\\'+S\\x08\\\\\\x07-\\x1f\\x10 !,*L)0**d),\\n--G!\\x15O)[Ȫ\\x00u\\x05\\x11T,fS%8)0!ps0e\\x07,\\x0c\\'7\\x0e\\x19g\\x08 g%\\x04\"(l*#R-c_)G)*\\x1adĭM\\x15̨߬#y ($쬮#9~)S\\x7f\\x0b#*\\x1fr\\x1fǡ5T\"+\\x19\"4\\'\\x17),\\x7fQ*d)\\t/T\\n\\x1c#ҭ¬\\r%\\x19s)릶(j*\\x00&@q9#\\x1dʬ*Y\\x1f8,(,*+\\x14)H&Eȡ\\x01\\n\\x17*#!\\t*l&(&((0gJ+\\x02)x\\x1c*\\x0f+Z.<(\\r(a$\\n( r`\\x1d}\\x11]1\\')N*\\x14%u+d(/\\x18\\x1fz\\x07ު\\r)!דɧ$+?(\\x10.\\x1f\\x12*\\x07*\\x05+%\\x00T˪]!\\x03$s+;Il*x%t% ,Z\\x06\\x0f=\\x18ȮC >\\',*ߣ3,\\x1a/*R$(t,ǩ\\\\$*~(;+2K& \\x1c\"uT\\x1d\\\\\\x11)Pfz$M)\\x15,\\x0b*b\\x1e', 'rating': '9'}, Document {'id': '7', 'payload': None, 'title': 'Gladiator', 'genre': 'action', 'description': 'A betrayed Roman general seeks revenge against the corrupt emperor who murdered his family.', 'vector': '.+\\x93;m^)\\x7f*\\x19\\x1a*3#\\x01\\x02.~$\\x02+$\\x1f|)e+X\\x04\\x04(C7,p\\x05\\x05M|+%.\\'*.50\\'~+(*$\\'t&\\x1a\\x11.\\x01\\x12\\'Ц+ (,\\x13\\'\\x1aWX\\';? \\x1c/Z+x\\x1d\\x1c\\x1a)\\x0f%jqT\\x14s*7\\x1e[*Fo,\\x0c0*hW*̠]-++h(o\\x1ebh)*\\x16\\x003\\x1clb!)*\\x0b\\x1dy.\\x1f@+,vV.,\\x197&(%[\\x1dQ(&\\'-(I#E!0$ЬU.X(-t(\\x1c&z$jED\\x1c(,$;\\x1cP*))\"),\\',(`^a$M5,\\x13\\x18a|(W9Ԯ\\x1e#i$\"\\x9d\\x08ޙ*\\x03!sì.l\\x00S\\x1f\\n$*\\x07!2k7\\'\\x1a$],V(ɪP0%##\\x1fj&\"z\\x0f).((,\\x0f,P#\\x01(G\\x1c./,\\x0bҡ$,\\'j,-˦)\\x08&h\\')&.\\x08.k(.v$̦\\'T\\x10$-Q(:r!(\\x07%O\\x1e\\'!,f-)*A\\x0f)B%t*}\\x0f,\\n.\\x1b-c\\x0c( .})\\x00\\x0c9&7\\x0f֬\\x1b\\x1f,!+\\r,\\x16%\\x1e\\\\\\x1cX\\x16\\x0f\\'ĥ\\x00-G\"\\x1e٥V, *̠!U,|+X\\',&m\\x02\\x01*,ʤQ\\x1d%,l\\x1a\\x15o*\\x12.(w#e\\x14', 'rating': '8'}, Document {'id': '8', 'payload': None, 'title': 'Inception', 'genre': 'action', 'description': 'A thief who enters dreams to steal secrets faces his toughest mission yet, with reality itself at stake.', 'vector': '**6$\\x18!\"&T+\\x1d١.$\"\\x0e3(%\\x01&*1\\x02**d\\')Щ\\uf805\\x1a-1*%+7էC \\x1c+ܡߡ\\x0f$\\x07**$~\\x1d(+ܫw($)[-\\x17\\x04+\\x0f(G2X-\\x1a+ףsp$%,hIf\\'\\x1cOg!-LA\\x10\\x0b#;\\n\\x03XR&,N-+Jx\\x1f;\\x00ܠ*!?#d(a&iAw\\x15Q.|\\'4*_8,a\\x14/Q8\\x0c*E9\\x1dW)m\\x1b\\x1d,N*&*-K*Wjh2\\'\\x11\\x15g&(\\x10)(\\x1aR\\x16\\x14\\x11),\\x00)9-\\x19*\\x17\\x19#-˨4\\x1aK\\x07ΩݦO&\\x02,\\x1f\\x0b7\\x05x(!\"-g%q\\x03@)\\':\\x07$\\x1d\\x00j-\\x16x\\'r,\\t,7\"\\x1e}*%)/\\r\\n*\\x06q #((\\n(N,o\\x16NE\\x1f߭M%aO-I[\\'y\\x11ħh\\\\\\\\,\\x1c,,\\x00-h)F)\\x03$\\x1b&s\\x1b()@+n%R\")(%`!.\\r)6-\\x11B&.!A((\\x13\\x1c,%J>*N(.ɫ\\x1b\\x08t$S\\x00{2+n*\\x02&W1K,*p\\'g)\\x19\\x18Ț)#-\"~\\'m# X0[&ɪd(\\x0e%6+)D&\\x13,\\x1d(,(\\x7f䧨,ʫ+\\x1eS\\'?\\x18\\x04-,m$M#4(^[\\x1c+\\x05\"\\x15\\x01', 'rating': '9'}, Document {'id': '9', 'payload': None, 'title': 'The Avengers', 'genre': 'action', 'description': \"Earth's mightiest heroes come together to stop an alien invasion that threatens the entire planet.\", 'vector': '&\\')\\x1b-0¨ȧ,=)\\x1c㞌)O\\x18\\'yu/4(-g+P!c\\x12\\x1dԧt$1\\x03*| m*\\x0fX&Ȭ[\\x1e\\x04\\x1cŬaC#p\\'Ϣ߫,w**\\x14ᠸ\\x1f$˥\\x1ff\\x1b*/\\x0c)\"\\x15<\\x14zc+,9%\\',[]&a\\x16@)4+$`-C+*ӣҪ&: w\\x195F&p \\x7f\\x1d>*0x*B\\x1d`**,)0\\x04\\t*$\\x07)|,x\\x12\\x00G\\x1b\\x1f(\\x1e&l\\x04-)C#_=)\\x0e(\\x1e$i(6\"C4\\x7f+\\x11(\\x03#!,Ξ\\x13J,Q-\"$&?(h+T,S\\x0c,j$d&+XT!f!\\x1b\\'\\x0c\\x1b%\\x18*\\x05+X)\\x1cΰ-%@\\x1f{--04\\t&$(\"X$w)\\x1cy zZ)y$)\\x02K\\x00\\x00˨\\x04\\x1a)\\x00/+)kr*\\x0f,\\x14?,:/\"M&`-(#\\ue840,#((@(\\x160\\x00-e!}C>\"()\\'\\x05+\\',\\x1d,\\x04w),m(a=+\\x1a-*\\'X,(\\x16ק!a,\\x18*+៥ڨs)\\x15?u\\x13$l\\x00~\\x1c,HX)(', 'rating': '8'}]}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res = client.ft(index_name).search(\"*\")\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Document {'id': '0', 'payload': None, 'title': 'Explosive Pursuit', 'genre': 'action', 'description': 'A daring cop chases a notorious criminal across the city in a high-stakes game of cat and mouse.', 'vector': '+S\\x18\\x1dsQ*)$8{.+oH3&\\x0f+ (O$n!\\x18*)%\\x17Eg+):*),\\x12YG*1\"a\\'o$\\x1d-\\ueb79\\x1f+f,0\\x1dgJ३T&T*U)(`\"N(d-ǭ(3)\\'H$ڪu\\n˨\\x1bz\"bu-hp(n.)Z*\\x06)P\\x17(&!\\x1d#2! o$u*)-& p*(\\'m\\nb/**$ Λ|#E$-\\x0080[̝Ν[+h%Cc.\\x07z+#X$34$(%s\\x1e\\x1a`T/\\x18q-\\x1c,!\\x0f%hH&.\\x0c)1DJ&\\x19.xf-\\x1b\\x0b\\x1e\\x18\\x17/Q!(o(7(ԭ&\\x1ez).,\\x10){\\'\\x1f+B)\\x1a\\x15#%ܩ\\x19%*(,짐$\\x1c+L#p)7*\\x00\\x00%}8\\x1e(\\'*ߡkA)6.(/v!J((Zhkˠ\\x1c,$ˢ\\x0c+\\x1dʨ+,Kˡw\\x04-$đD!^*&:@\"%\\x0c(ު\\x1f {\\x07(.\\x1c@.\\x1e[)߬,\"5(\\x1f([\\x00|7\\x1e+19A-\\x16%\\x1f$\\x08,KX\\x1d*.m-ۢ寋\\x1f)\\x17v(*\"3%N)&\\x1f\\x1f\\x02\\x15\\x04\\x16,\\x9ey!Z+\\x12|\\x08$Uz%\\x1cRh\\x0c\\x18(#N\\x1dGJdÕX\\x0c\\x19蜗./\\x1b\"̘%)b+**.Ϫ\\x00 @,_-\\x042y+.1\\x1fS{)d\\x00\\x00o\\x1eɪ:\\x7f,\\x08w\"] \\x1c.W\\'!\\x00)Ѧ\\x06\\'\\x04-\\x1b\\x17.L %\\x1c,6$+M\\x1du\\x1f\\'e0#*(z)V,$\\x02ܬ\"&p%\\r2\\x1f-D\\'t*9\\x03ά%˩%$\\x1aW)%,<\\x15,!%\\x1f!k\\x1e&\\x16*)\\x17\\x1e((u*(>\\x03[\\x1cc,%\\x003p(w*\\x16\\x1f4 =)+]\\x14\\x18\\x1a#\\x1e,G\\x1f!\\'ڦ\\',\\x1c\\x1f)\\t$j/)N)\\x16(\\x0co+t? \\x06.-ר+\\x18,\\x18ԫf*O)\\n kũ`*)\\x0f\\'m\\\\%B#\\x17)', 'rating': '8'},\n", - " Document {'id': '2', 'payload': None, 'title': 'Fast & Furious 9', 'genre': 'action', 'description': 'Dom and his crew face off against a high-tech enemy with advanced weapons and technology.', 'vector': '=\\x11-A\\x1d*+\\x01-g4);(m+\\x1c\\rT,\\x1e+b#\\x1bء(\\x18\\x00.q-uڥL(֦\\x08>}-\\x1d*O¬P \\x18\\x05)J\\x1a,\\x1b*!)\\x19&\\x14R,A#Wo&ʡA+\\r+),-\\x1dU#ߤ&&\\x1fᗬ/۩?*3˕~+Ȩ._)s&\\x02.\\x1c\\x11-,$%<+)\\x04\\'(,d$1g3*![%8\\t5)V1(LӨ|.-*\\x00\\x19\"Z+(* c-\\x19ǩ$A\\x1d\\x1c*ޣ\\t\\x16M\\'#E%,l\\x155\\x1f)-\\x00\\x15,\\x10*1-(ͥȫj\\x11I$\\x13\\'\"+\\x19%˨#\\'\\'\\'\\u2438-(**,*˦[t)r\\x02)H*U(먤%p\"$#$\\x1b$\\x0eb\\'AC)?\\x1aD\\x14dާ\\x00\\x008b& u*z\\x1f,+],)Bp5r,\\x0c\\x12,S\\x1c(1y\\',)Z*p(\\x19+\\x07$e 0*ө_\\x0b,Ĥ~7.vQ\\x1c\\x15&~) c,G,\\x1b!;*\\x07\\x1d\\x10(J-\\x1b,/.ά*ik) \\x1cp*k(\\x1e¨-)c/ɝ\\x12\\\\w{\\x1f\\x1c9\\x00dG\\x01\\x1f&۩h,A\\x1d$ -h\\'R+,\\x0bcԮ\\x08\\x1d).(o%-z&,-; \\x1c0\\x13*Μ,ҭ*H\\'v,K-\\x90\\x02\\x13\\x1d\\x04)$-Ir$B 9\\x15,R(', 'rating': '6'},\n", - " Document {'id': '3', 'payload': None, 'title': 'Black Widow', 'genre': 'action', 'description': 'Natasha Romanoff confronts her dark past and family ties as she battles a new enemy.', 'vector': '/~\\x17+%\\x04-Z(Δ\\x1f?\\x1c\\x16$6:B\\x14$\\x1c\\r\\x1ff\\x0b#)\\x06\\'(6\\x1c&\\x13),\"&$*-y)&)E*R\\x00u\\x16$.\\x1d\\x07\\x14V\\x1dXn)-.)\\x1d\\'\\'*]J\\x13)\\'%{,Y\\x07*+D˪\\x11..(%\\x15))\"M!(\\x17\\n!R+\\x02E\\x1f ^,\\'&Ց$\\x10**\\x0f1(ڨ)rڬ)+&\\x17.\\x01 /\\x18oK,J)[*\\x06%T\\x1f8\\x00N#%K\\x16/(!)r(}\\x1eV,x`)d*j(#/z\\x1d/0\\x13?ӧ\\n(6(}$*%\\'-$d*\\'(\\x06\\x14\\x1e\\x13`(JY4#v,4T!<\\x02\\x00*$\\x17-\\'i5*\\x14+\\x18\\x19ҥi&($.\\x1c t\\x1a!)\\x0f,6۬7\\'A*\\x04\\x1f\\x08(\\x02\\x05)|&\\x00-k\\x7f&,⡡X.P!\\x0b)!(2(#Ѫ.\\x1e\\x94\\x10\\x1a\\x06`,j0}\\x15oG\\x06\\x1b\\'*0*ˮ\\'$8\\x0b-?-m/u)0\\x0eXt*~,ί)\\x1f,/D\\x10%v\\x00zl\\x1f٩j-̦2\\x1b,]$\\x0b\\'$)F\\',\\x13/&}.E(gK\\x14L\\x00硇R\\'Ȭժ+\\x1bb\\x1a\\x1d*\\x05\"j)#Ϊ1.k-7we\\x13\\x05\\'g\\x15rΥ\\t,)%!Ҩy-%f\\x1fwg*P,hRw\"*+1<\\x18\\x1c\\x1a)m(8)\"\\x0c*%,h*U', 'rating': '7'},\n", - " Document {'id': '4', 'payload': None, 'title': 'John Wick', 'genre': 'action', 'description': 'A retired hitman seeks vengeance against those who wronged him, leaving a trail of destruction in his wake.', 'vector': '\\x9b).\\x19$\\x1b\\r#Mդk\\'\\x03H(p\\x1f%2$.l$@:-/\\x11QSѥ\\x03*%թţ\\x1cޭO$*H#%%h$ʩ\\x0f!C+\\x00*fa((h(kx-y\\x1ch*ĢQ()d%\\\\,U(j,)E$\\x0e|$%,Q!^-* #*K+)$%1\\x10,˧*k.؟l5#()$ѯ\\x15,\\x0eϘd(\\x1c,窺;,4(!k\\x1d\\x19,\\x1d\\x00(&& \\x00y,#\\n\\'w%\\x16&a)&*y\\x11/$\"?#,ެP\\x08.ǥ(\\x17g#(5xi/,U*!#a(u(\\x0c\\x16\\x19T$.h1!0%%w\\x01,[\">N\\'A(4\\x19+\\x03Y\\x1f.\"\\x1e\\n:\\x1d-<\\x1e)\\x00P)D*ڦ\\x15%&ʨե&\\'\\x05\\x01\\t&+$$p,=\\x00\\x18#\\')\\x03,\\x1e,,t4>1/\\x1f#\\x10QM\"=-\\x0b̟],2\\x0b#)H-9.& V#h\\x1e\\'$\\x02$+0\\x16+.\\x06)\\'\\'-\\'d*E\\x1eŬ \\x061\\x14R)ƩVW*\\x10XI&ͫ\\x04\"#\\'ҭ\\x17,\\x0f\\x1fB)#\\'!ӟ\\x00\\x04ꕔ%\\x12 +!]m*\\x19%o--s\\x1e$(Nάܪ\\x1cq\"(,N*&\\x1eB) \\x0e(y%p)+\\x10?A,(\\'3ǮW $wt)+\"\\x0c)t)yY(!Ƥ0\\x0536\"', 'rating': '8'},\n", - " Document {'id': '6', 'payload': None, 'title': 'The Dark Knight', 'genre': 'action', 'description': 'Batman faces off against the Joker, a criminal mastermind who threatens to plunge Gotham into chaos.', 'vector': '\\n/)j &*,\\'l$rR(\\x1d;ѥZ%, (*]-|\\x1f\\x03k+v(+$\\x1d)֭(6#բ0E+0() =&$9Y+*}\\x12+m(\\x1c(\\x11$I!V.()\"\\'l)J),\\x12*O Р,V\\'X$Ϥ&\\'\\x1a,\\x11\\')\\'N*ʠ1\\'\\x1c-::DJ)%\\'}(4\\x1fg\\'+S\\x08\\\\\\x07-\\x1f\\x10 !,*L)0**d),\\n--G!\\x15O)[Ȫ\\x00u\\x05\\x11T,fS%8)0!ps0e\\x07,\\x0c\\'7\\x0e\\x19g\\x08 g%\\x04\"(l*#R-c_)G)*\\x1adĭM\\x15̨߬#y ($쬮#9~)S\\x7f\\x0b#*\\x1fr\\x1fǡ5T\"+\\x19\"4\\'\\x17),\\x7fQ*d)\\t/T\\n\\x1c#ҭ¬\\r%\\x19s)릶(j*\\x00&@q9#\\x1dʬ*Y\\x1f8,(,*+\\x14)H&Eȡ\\x01\\n\\x17*#!\\t*l&(&((0gJ+\\x02)x\\x1c*\\x0f+Z.<(\\r(a$\\n( r`\\x1d}\\x11]1\\')N*\\x14%u+d(/\\x18\\x1fz\\x07ު\\r)!דɧ$+?(\\x10.\\x1f\\x12*\\x07*\\x05+%\\x00T˪]!\\x03$s+;Il*x%t% ,Z\\x06\\x0f=\\x18ȮC >\\',*ߣ3,\\x1a/*R$(t,ǩ\\\\$*~(;+2K& \\x1c\"uT\\x1d\\\\\\x11)Pfz$M)\\x15,\\x0b*b\\x1e', 'rating': '9'},\n", - " Document {'id': '7', 'payload': None, 'title': 'Gladiator', 'genre': 'action', 'description': 'A betrayed Roman general seeks revenge against the corrupt emperor who murdered his family.', 'vector': '.+\\x93;m^)\\x7f*\\x19\\x1a*3#\\x01\\x02.~$\\x02+$\\x1f|)e+X\\x04\\x04(C7,p\\x05\\x05M|+%.\\'*.50\\'~+(*$\\'t&\\x1a\\x11.\\x01\\x12\\'Ц+ (,\\x13\\'\\x1aWX\\';? \\x1c/Z+x\\x1d\\x1c\\x1a)\\x0f%jqT\\x14s*7\\x1e[*Fo,\\x0c0*hW*̠]-++h(o\\x1ebh)*\\x16\\x003\\x1clb!)*\\x0b\\x1dy.\\x1f@+,vV.,\\x197&(%[\\x1dQ(&\\'-(I#E!0$ЬU.X(-t(\\x1c&z$jED\\x1c(,$;\\x1cP*))\"),\\',(`^a$M5,\\x13\\x18a|(W9Ԯ\\x1e#i$\"\\x9d\\x08ޙ*\\x03!sì.l\\x00S\\x1f\\n$*\\x07!2k7\\'\\x1a$],V(ɪP0%##\\x1fj&\"z\\x0f).((,\\x0f,P#\\x01(G\\x1c./,\\x0bҡ$,\\'j,-˦)\\x08&h\\')&.\\x08.k(.v$̦\\'T\\x10$-Q(:r!(\\x07%O\\x1e\\'!,f-)*A\\x0f)B%t*}\\x0f,\\n.\\x1b-c\\x0c( .})\\x00\\x0c9&7\\x0f֬\\x1b\\x1f,!+\\r,\\x16%\\x1e\\\\\\x1cX\\x16\\x0f\\'ĥ\\x00-G\"\\x1e٥V, *̠!U,|+X\\',&m\\x02\\x01*,ʤQ\\x1d%,l\\x1a\\x15o*\\x12.(w#e\\x14', 'rating': '8'},\n", - " Document {'id': '8', 'payload': None, 'title': 'Inception', 'genre': 'action', 'description': 'A thief who enters dreams to steal secrets faces his toughest mission yet, with reality itself at stake.', 'vector': '**6$\\x18!\"&T+\\x1d١.$\"\\x0e3(%\\x01&*1\\x02**d\\')Щ\\uf805\\x1a-1*%+7էC \\x1c+ܡߡ\\x0f$\\x07**$~\\x1d(+ܫw($)[-\\x17\\x04+\\x0f(G2X-\\x1a+ףsp$%,hIf\\'\\x1cOg!-LA\\x10\\x0b#;\\n\\x03XR&,N-+Jx\\x1f;\\x00ܠ*!?#d(a&iAw\\x15Q.|\\'4*_8,a\\x14/Q8\\x0c*E9\\x1dW)m\\x1b\\x1d,N*&*-K*Wjh2\\'\\x11\\x15g&(\\x10)(\\x1aR\\x16\\x14\\x11),\\x00)9-\\x19*\\x17\\x19#-˨4\\x1aK\\x07ΩݦO&\\x02,\\x1f\\x0b7\\x05x(!\"-g%q\\x03@)\\':\\x07$\\x1d\\x00j-\\x16x\\'r,\\t,7\"\\x1e}*%)/\\r\\n*\\x06q #((\\n(N,o\\x16NE\\x1f߭M%aO-I[\\'y\\x11ħh\\\\\\\\,\\x1c,,\\x00-h)F)\\x03$\\x1b&s\\x1b()@+n%R\")(%`!.\\r)6-\\x11B&.!A((\\x13\\x1c,%J>*N(.ɫ\\x1b\\x08t$S\\x00{2+n*\\x02&W1K,*p\\'g)\\x19\\x18Ț)#-\"~\\'m# X0[&ɪd(\\x0e%6+)D&\\x13,\\x1d(,(\\x7f䧨,ʫ+\\x1eS\\'?\\x18\\x04-,m$M#4(^[\\x1c+\\x05\"\\x15\\x01', 'rating': '9'},\n", - " Document {'id': '9', 'payload': None, 'title': 'The Avengers', 'genre': 'action', 'description': \"Earth's mightiest heroes come together to stop an alien invasion that threatens the entire planet.\", 'vector': '&\\')\\x1b-0¨ȧ,=)\\x1c㞌)O\\x18\\'yu/4(-g+P!c\\x12\\x1dԧt$1\\x03*| m*\\x0fX&Ȭ[\\x1e\\x04\\x1cŬaC#p\\'Ϣ߫,w**\\x14ᠸ\\x1f$˥\\x1ff\\x1b*/\\x0c)\"\\x15<\\x14zc+,9%\\',[]&a\\x16@)4+$`-C+*ӣҪ&: w\\x195F&p \\x7f\\x1d>*0x*B\\x1d`**,)0\\x04\\t*$\\x07)|,x\\x12\\x00G\\x1b\\x1f(\\x1e&l\\x04-)C#_=)\\x0e(\\x1e$i(6\"C4\\x7f+\\x11(\\x03#!,Ξ\\x13J,Q-\"$&?(h+T,S\\x0c,j$d&+XT!f!\\x1b\\'\\x0c\\x1b%\\x18*\\x05+X)\\x1cΰ-%@\\x1f{--04\\t&$(\"X$w)\\x1cy zZ)y$)\\x02K\\x00\\x00˨\\x04\\x1a)\\x00/+)kr*\\x0f,\\x14?,:/\"M&`-(#\\ue840,#((@(\\x160\\x00-e!}C>\"()\\'\\x05+\\',\\x1d,\\x04w),m(a=+\\x1a-*\\'X,(\\x16ק!a,\\x18*+៥ڨs)\\x15?u\\x13$l\\x00~\\x1c,HX)(', 'rating': '8'}]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res.docs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Does work with pure redis client.\n", - "client.memory_usage('rvl:37f6b0ea780c4a84a156f35f54a31c7e')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}