I’m able to use Pinecone as a vector database to store embeddings created using OpenAI text-embedding-ada-002, and I create a ConversationalRetrievalChain using langchain, where I pass OpenAI gpt-3.5-turbo as the LLM, and the Pinecone vectorstore as the retriever. The use case is that I’m saving the backstory of a fictional company employee so that I can do question and answer using embeddings.
I want to now save three different backstories into the same Pinecone index, which I suppose can be accomplished using namespaces? How can I change the code to save and retrieve from three different namespaces? I searched langchain’s documentation on its Pinecone vectorstore, but I couldn’t find anything.
Relevant sections of my code:
import streamlit as st
from dotenv import load_dotenv
import re, os
import pinecone
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Pinecone
from langchain.chat_models import ChatOpenAI
from langchain.memory import ConversationBufferMemory
from langchain.chains import ConversationalRetrievalChain
from langchain.callbacks import get_openai_callback
from langchain.prompts import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
def initializePinecone():
PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY")
PINECONE_ENVIRONMENT = os.environ.get("PINECONE_ENVIRONMENT")
pinecone.init(api_key=PINECONE_API_KEY,
environment=PINECONE_ENVIRONMENT)
# Set the index name
index_name = "fictionalemployees"
# check if the extractive-question-answering index exists
if index_name not in pinecone.list_indexes():
# create the index if it does not exist
pinecone.create_index(
index_name,
dimension=1536,
metric="cosine"
)
return index_name
def get_pineconeVectorStore(text_chunks, indexName):
embeddings = OpenAIEmbeddings(deployment="text-embedding-ada-002", model="text-embedding-ada-002", chunk_size=10)
pineconeVectorStore = Pinecone.from_texts(
text_chunks,
embeddings,
index_name=indexName)
return pineconeVectorStore
def get_conversation_chain(vectorstore):
llm = ChatOpenAI(model="gpt-3.5-turbo", max_tokens=50)
general_system_template = r"""
You are John, a Customer Service Officer at the company. You are answering questions regarding your job at helping with the rental application process.
----
{context}
----
"""
general_user_template = "Question:{question}"
messages = [
SystemMessagePromptTemplate.from_template(general_system_template),
HumanMessagePromptTemplate.from_template(general_user_template)
]
qa_prompt = ChatPromptTemplate.from_messages( messages )
memory = ConversationBufferMemory(
memory_key='chat_history', return_messages=True)
conversation_chain = ConversationalRetrievalChain.from_llm(
llm=llm,
retriever=vectorstore.as_retriever(),
memory=memory,
combine_docs_chain_kwargs={"prompt": qa_prompt}
)
return conversation_chain
def handle_userinput(user_question):
response = count_tokens(
st.session_state.conversation,
{'question': user_question})
st.session_state.chat_history = response['chat_history']
for i, message in enumerate(st.session_state.chat_history):
if i % 2 == 0:
st.write(user_template.replace(
"{{MSG}}", message.content), unsafe_allow_html=True)
else:
message.content = truncate_at_last_sentence(replace_list_format(message.content))
st.write(bot_template.replace(
"{{MSG}}", message.content), unsafe_allow_html=True)
# Somewhere in main:
#create Pinecone vector store
pineconeVectorStore = get_pineconeVectorStore(
text_chunks, pineconeIndexName)
# create conversation chain using Pinecone
st.session_state.conversation = get_conversation_chain(
pineconeVectorStore)
nameSpace:smiley:, and roughly shown how it needs to be used.