refactor: fix breaking changes for langchain==0.1.14 and pinecone-client==3.2.2

Author: lpm0073

Makefile

@@ -20,7 +20,7 @@ PINECONE_VECTORSTORE_TEXT_KEY=lc_id\n\
 PINECONE_METRIC=dotproduct\n\
 PINECONE_DIMENSIONS=1536\n\
 OPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\n\
-OPENAI_PROMPT_MODEL_NAME=text-davinci-003\n\
+OPENAI_PROMPT_MODEL_NAME=gpt-3.5-turbo-instruct\n\
 OPENAI_CHAT_TEMPERATURE=0.0\n\
 OPENAI_CHAT_MAX_RETRIES=3\n\
 DEBUG_MODE=True\n" >> .env)

README.md

@@ -136,7 +136,7 @@ OPENAI_API_KEY=PLEASE-ADD-ME
 OPENAI_CHAT_MAX_RETRIES=3
 OPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo
 OPENAI_CHAT_TEMPERATURE=0.0
-OPENAI_PROMPT_MODEL_NAME=text-davinci-003
+OPENAI_PROMPT_MODEL_NAME=gpt-3.5-turbo-instruct
 
 # Pinecone API
 PINECONE_API_KEY=PLEASE-ADD-ME

models/conf.py

@@ -87,7 +87,7 @@ class SettingsDefaults:
     OPENAI_ENDPOINT_IMAGE_SIZE = "1024x768"
     OPENAI_CHAT_CACHE = True
     OPENAI_CHAT_MODEL_NAME = "gpt-3.5-turbo"
-    OPENAI_PROMPT_MODEL_NAME = "text-davinci-003"
+    OPENAI_PROMPT_MODEL_NAME = "gpt-3.5-turbo"
     OPENAI_CHAT_TEMPERATURE = 0.0
     OPENAI_CHAT_MAX_RETRIES = 3
 

models/hybrid_search_retreiver.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=E0611,E1101
 """
 Hybrid Search Retriever. A class that combines the following:
     - OpenAI prompting and ChatModel
@@ -22,18 +23,20 @@
 
 # pinecone integration
 from langchain.cache import InMemoryCache
-from langchain.chat_models import ChatOpenAI
 
 # embedding
 from langchain.globals import set_llm_cache
 
 # prompting and chat
 from langchain.llms.openai import OpenAI
 from langchain.prompts import PromptTemplate
+from langchain.schema import BaseMessage, HumanMessage, SystemMessage
+from langchain_community.chat_models import ChatOpenAI
 
 # hybrid search capability
-from langchain.retrievers import PineconeHybridSearchRetriever
-from langchain.schema import BaseMessage, HumanMessage, SystemMessage
+from langchain_community.retrievers.pinecone_hybrid_search import (
+    PineconeHybridSearchRetriever,
+)
 from pinecone_text.sparse import BM25Encoder  # pylint: disable=import-error
 
 # this project

models/pinecone.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=E0611,E1101
 """A class to manage the lifecycle of Pinecone vector database indexes."""
 
 # document loading
@@ -9,12 +10,16 @@
 import logging
 import os
 
-# pinecone integration
-import pinecone
-from langchain.document_loaders import PyPDFLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.text_splitter import Document
+# from langchain.text_splitter import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores.pinecone import Pinecone as LCPinecone
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.embeddings import OpenAIEmbeddings
+
+# pinecone integration
+# import pinecone
+from pinecone import Pinecone, ServerlessSpec
+from pinecone.core.client.exceptions import PineconeApiException
 
 # this project
 from models.conf import settings
@@ -24,28 +29,29 @@
 
 
 # pylint: disable=too-few-public-methods
-class TextSplitter:
-    """
-    Custom text splitter that adds metadata to the Document object
-    which is required by PineconeHybridSearchRetriever.
-    """
-
-    def create_documents(self, texts):
-        """Create documents"""
-        documents = []
-        for text in texts:
-            # Create a Document object with the text and metadata
-            document = Document(page_content=text, metadata={"context": text})
-            documents.append(document)
-        return documents
+# class TextSplitter:
+#     """
+#     Custom text splitter that adds metadata to the Document object
+#     which is required by PineconeHybridSearchRetriever.
+#     """
+
+#     def create_documents(self, texts):
+#         """Create documents"""
+#         documents = []
+#         for text in texts:
+#             # Create a Document object with the text and metadata
+#             document = Document(page_content=text, metadata={"context": text})
+#             documents.append(document)
+#         return documents
 
 
 class PineconeIndex:
     """Pinecone helper class."""
 
-    _index: pinecone.Index = None
+    _pinecone = None
+    _index: Pinecone.Index = None
     _index_name: str = None
-    _text_splitter: TextSplitter = None
+    _text_splitter: RecursiveCharacterTextSplitter = None
     _openai_embeddings: OpenAIEmbeddings = None
     _vector_store: LCPinecone = None
 
@@ -69,11 +75,11 @@ def index_name(self, value: str) -> None:
             self.init_index()
 
     @property
-    def index(self) -> pinecone.Index:
+    def index(self) -> Pinecone.Index:
         """pinecone.Index lazy read-only property."""
         if self._index is None:
             self.init_index()
-            self._index = pinecone.Index(index_name=self.index_name)
+            self._index = self.pinecone.Index(name=self.index_name)
         return self._index
 
     @property
@@ -85,7 +91,7 @@ def index_stats(self) -> dict:
     @property
     def initialized(self) -> bool:
         """initialized read-only property."""
-        indexes = pinecone.manage.list_indexes()
+        indexes = self.pinecone.list_indexes()
         return self.index_name in indexes
 
     @property
@@ -113,23 +119,31 @@ def openai_embeddings(self) -> OpenAIEmbeddings:
         return self._openai_embeddings
 
     @property
-    def text_splitter(self) -> TextSplitter:
-        """TextSplitter lazy read-only property."""
+    def pinecone(self):
+        """Pinecone lazy read-only property."""
+        if self._pinecone is None:
+            self._pinecone = Pinecone(api_key=settings.pinecone_api_key.get_secret_value())
+        return self._pinecone
+
+    @property
+    def text_splitter(self) -> RecursiveCharacterTextSplitter:
+        """lazy read-only property."""
         if self._text_splitter is None:
-            self._text_splitter = TextSplitter()
+            self._text_splitter = RecursiveCharacterTextSplitter()
         return self._text_splitter
 
     def init_index(self):
         """Verify that an index named self.index_name exists in Pinecone. If not, create it."""
-        indexes = pinecone.manage.list_indexes()
+        indexes = self.pinecone.list_indexes()
         if self.index_name not in indexes:
             logging.debug("Index does not exist.")
             self.create()
 
     def init(self):
         """Initialize Pinecone."""
         # pylint: disable=no-member
-        pinecone.init(api_key=settings.pinecone_api_key.get_secret_value(), environment=settings.pinecone_environment)
+
+        # pinecone.init(api_key=settings.pinecone_api_key.get_secret_value(), environment=settings.pinecone_environment)
         self._index = None
         self._index_name = None
         self._text_splitter = None
@@ -142,23 +156,30 @@ def delete(self):
             logging.debug("Index does not exist. Nothing to delete.")
             return
         print("Deleting index...")
-        pinecone.delete_index(self.index_name)
+        self.pinecone.delete_index(self.index_name)
 
     def create(self):
         """Create index."""
-        metadata_config = {
-            "indexed": [settings.pinecone_vectorstore_text_key, "lc_type"],
-            "context": ["lc_text"],
-        }
+        # deprecated?
+        # metadata_config = {
+        #     "indexed": [settings.pinecone_vectorstore_text_key, "lc_type"],
+        #     "context": ["lc_text"],
+        # }
         print("Creating index. This may take a few minutes...")
-
-        pinecone.create_index(
-            name=self.index_name,
-            dimension=settings.pinecone_dimensions,
-            metric=settings.pinecone_metric,
-            metadata_config=metadata_config,
+        serverless_spec = ServerlessSpec(
+            cloud="aws",
+            region="us-west-2",
         )
-        print("Index created.")
+        try:
+            self.pinecone.create_index(
+                name=self.index_name,
+                dimension=settings.pinecone_dimensions,
+                metric=settings.pinecone_metric,
+                spec=serverless_spec,
+            )
+            print("Index created.")
+        except PineconeApiException:
+            pass
 
     def initialize(self):
         """Initialize index."""

models/tests/mock_data/.env.test_01

@@ -10,6 +10,6 @@
     OPENAI_ENDPOINT_IMAGE_SIZE = "TEST_1024x768"
     OPENAI_CHAT_CACHE = False
     OPENAI_CHAT_MODEL_NAME = "TEST_gpt-3.5-turbo"
-    OPENAI_PROMPT_MODEL_NAME = "TEST_text-davinci-003"
+    OPENAI_PROMPT_MODEL_NAME = "TEST_gpt-3.5-turbo-instruct"
     OPENAI_CHAT_TEMPERATURE = 1.0
     OPENAI_CHAT_MAX_RETRIES = 5

models/tests/test_hsr.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=E0611,E1101
 # flake8: noqa: F401
 """
 Test integrity of base class.
 """
 import pytest  # pylint: disable=unused-import
-from langchain.chat_models import ChatOpenAI
+from langchain_community.chat_models import ChatOpenAI
 
 from models.hybrid_search_retreiver import HybridSearchRetriever
 from models.pinecone import PineconeIndex

models/yt.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=E0611
 """
     LangChain Quickstart
     ~~~~~~~~~~~~~~~~~~~~
@@ -15,8 +16,6 @@
 # 5.) sequential chains
 # 4.) chains
 from langchain.chains import LLMChain, SimpleSequentialChain
-from langchain.chat_models import ChatOpenAI
-from langchain.embeddings import OpenAIEmbeddings
 
 # 1.) wrappers
 from langchain.llms.openai import OpenAI
@@ -33,6 +32,8 @@
 
 # 7.) pinecode client
 from langchain.vectorstores.pinecone import Pinecone
+from langchain_community.chat_models import ChatOpenAI
+from langchain_community.embeddings import OpenAIEmbeddings
 
 # 8.) LangChain agents
 from langchain_experimental.agents.agent_toolkits.python.base import create_python_agent
@@ -75,14 +76,14 @@ class LangChainDev:
     def test_01_basic(self):
         """Test a basic request"""
 
-        llm = OpenAI(model_name="text-davinci-003")
+        llm = OpenAI(model_name="gpt-3.5-turbo-instruct")
         retval = llm("explain large language models in one sentence")
         print(retval)
 
     # 2.) models and messages. minute 6:08
     def test_02_chat_model(self):
         """Test a chat model"""
-        chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.3)
+        chat = ChatOpenAI(model_name="gpt-3.5-turbo-instruct", temperature=0.3)
         messages = [
             SystemMessage(content="You are an expert data scientist"),
             HumanMessage(content="Write a Python script that trains a neural network on simulated data"),
@@ -102,7 +103,7 @@ def get_prompt(self):
 
     def test_03_prompt_templates(self):
         """Test prompt templates"""
-        llm = OpenAI(model_name="text-davinci-003")
+        llm = OpenAI(model_name="gpt-3.5-turbo-instruct")
         prompt = self.get_prompt()
         retval = llm(prompt.format(concept="regularization"))
         print(retval)
@@ -115,7 +116,7 @@ def get_chain(self, llm, prompt):
 
     def test_04_chain(self):
         """Test a chain"""
-        llm = OpenAI(model_name="text-davinci-003")
+        llm = OpenAI(model_name="gpt-3.5-turbo-instruct")
         prompt = self.get_prompt()
         chain = self.get_chain(llm=llm, prompt=prompt)
         print(chain.run("autoencoder"))
@@ -137,7 +138,7 @@ def get_prompt_two(self):
 
     def get_explanation(self):
         """Get an explanation"""
-        llm = OpenAI(model_name="text-davinci-003")
+        llm = OpenAI(model_name="gpt-3.5-turbo-instruct")
         prompt = self.get_prompt()
         chain_one = self.get_chain(llm=llm, prompt=prompt)