OpenAI Function Calling and RAG Not working together in a single prompt?

We are currently in the process of developing a chatbot using LangChain, incorporating the RAG model for localization. Additionally, we aim to enhance user interaction by integrating the Function Calling feature of OpenAI to invoke certain APIs during conversations.

However, we have encountered an issue when attempting to combine these two functionalities. Despite using the ConversationBufferMemory to store chat history with the AgentExecutor, we are experiencing difficulty in obtaining the desired results.

First of all, the chatbot is able to provide responses when questions are asked that have answers within the local document. However, it fails to respond to follow-up questions, even though the chat history is available.

Furthermore, when we use the {context} part in the prompt, particularly with functions that have multiple arguments, the chatbot attempts to guess the missing arguments although instructed not to. However, this behavior is inconsistent and unreliable.

We have experimented with various prompt descriptions in an attempt to address this issue. However, we have found that either the RAG model works properly or the Function Calling feature performs adequately, but not both simultaneously.

Any light regarding this issue ?

Here’s my code:

# **Dependencies**

!pip install  cohere tiktoken langchain \
              openai==0.28.0 \
              pinecone-client==2.2.4 \
              docarray==0.39.0 \
              pydantic==1.10.8

from google.colab import userdata

# **VectoreStore from Dataset**

import pandas as pd
data = pd.read_json('testDataset.jsonl', lines=True)
data

import pinecone

# get API key from app.pinecone.io and environment from console
pinecone.init(
    api_key=userdata.get('PINECONE_API_KEY'),
    environment="gcp-starter"
)

import time

# pinecone.delete_index("llama-2-rag")

index_name = 'llama-2-rag'

if index_name not in pinecone.list_indexes():
    pinecone.create_index(
        index_name,
        dimension=1536,
        metric='cosine'
    )
    # wait for index to finish initialization
    while not pinecone.describe_index(index_name).status['ready']:
        time.sleep(1)

index = pinecone.Index(index_name)

from langchain.embeddings.openai import OpenAIEmbeddings

embed_model = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=userdata.get("OpenAI_API_KEY"))

from tqdm.auto import tqdm  # for progress bar

# data = dataset.to_pandas()  # this makes it easier to iterate over the dataset

batch_size = 100

for i in tqdm(range(0, len(data), batch_size)):
    i_end = min(len(data), i+batch_size) # 13 100
    # get batch of data
    batch = data.iloc[i:i_end]
    # generate unique ids for each chunk
    ids = [f"{x['chunk-id']}" for i, x in batch.iterrows()]
    # get text to embed
    texts = [x['chunk'] for _, x in batch.iterrows()]
    # embed text
    embeds = embed_model.embed_documents(texts)
    # get metadata to store in Pinecone
    metadata = [
        {'text': x['chunk'],
         'title': x['title']} for i, x in batch.iterrows()
    ]
    # add to Pinecone
    print(ids,embeds, metadata)
    try:
      index.upsert(vectors=zip(ids, embeds, metadata))
      print("Inserted")
    except Exception as e:
      print("got exception" + str(e))

print(index)

index.describe_index_stats()

from langchain.vectorstores import Pinecone

text_field = "text"  # the metadata field that contains our text

# initialize the vector store object
vectorstore = Pinecone(
    index, embed_model.embed_query, text_field
)

# **Retreiver in action**

retreiver=vectorstore.as_retriever()

retreiver.get_relevant_documents("what is the capital of Bangladesh ? ",k=2)

# **Function Definition as Commands..**


from langchain.tools import tool
import requests
from pydantic import BaseModel, Field, constr
import datetime
from datetime import date

# Define the input schema
class OpenMeteoInput(BaseModel):
    latitude: float = Field(..., description="Latitude of the location to fetch weather data for")
    longitude: float = Field(..., description="Longitude of the location to fetch weather data for")

@tool(args_schema=OpenMeteoInput)
def get_current_temperature(latitude: float, longitude: float) -> dict:
    """Fetch current Weather for given cities or coordinates. For example: what is the weather of Colombo ?"""

    BASE_URL = "https://api.open-meteo.com/v1/forecast"

    # Parameters for the request
    params = {
        'latitude': latitude,
        'longitude': longitude,
        'hourly': 'temperature_2m',
        'forecast_days': 1,
    }

    # Make the request
    response = requests.get(BASE_URL, params=params)

    if response.status_code == 200:
        results = response.json()
    else:
        raise Exception(f"API Request failed with status code: {response.status_code}")

    current_utc_time = datetime.datetime.utcnow()
    time_list = [datetime.datetime.fromisoformat(time_str.replace('Z', '+00:00')) for time_str in results['hourly']['time']]
    temperature_list = results['hourly']['temperature_2m']

    closest_time_index = min(range(len(time_list)), key=lambda i: abs(time_list[i] - current_utc_time))
    current_temperature = temperature_list[closest_time_index]

    return f'The current temperature is {current_temperature}°C'


class book_room_input(BaseModel):
    room_type: str = Field(..., description="Which type of room AC or Non-AC")
    class_type: str = Field(...,description="Which class of room it is. Business class or Economic class")
    check_in_date: date = Field(...,description="The date user will check-in")
    check_out_date: date = Field(...,description="The date user will check-out")
    mobile_no : constr(regex=r'(^(?:\+?88)?01[3-9]\d{8})$') = Field(...,description="Mobile number of the user")

@tool(args_schema=book_room_input)
def book_room(room_type: str, class_type: str, check_in_date: date, check_out_date: date, mobile_no: constr) -> str:
  """
    Book a room with the specified details.

    Args:
        room_type (str): Which type of room to book (AC or Non-AC).
        class_type (str): Which class of room it is (Business class or Economic class).
        check_in_date (date): The date the user will check-in.
        check_out_date (date): The date the user will check-out.
        mobile_no (str): Mobile number of the user.

    Returns:
        str: A message confirming the room booking.
    """
    # Placeholder logic for booking the room
  return f"Room has been booked for {room_type} {class_type} class from {check_in_date} to {check_out_date}. Mobile number: {mobile_no}."

class requestFoodFromRestaurant(BaseModel):
    item_name : str = Field(..., description="The food item they want to order from the restaurant")
    room_number: int = Field(..., description="Room number where the request is made")
    dine_in_type : str = Field(..., description="If the customer wants to eat in the door-step, take parcel or dine in the restaurant. It can have at most 3 values 'dine-in-room', 'dine-in-restaurant', 'parcel'")

@tool(args_schema=requestFoodFromRestaurant)
def order_resturant_item(item_name : str, room_number : int, dine_in_type : str) -> str:
  """
  Order food and bevarages at the hotel restaurant with item name and dine in type from a room.

  Args:
    item_name (str) : The food item they want to order from the restaurant
    room_number (int) : The room number from which the customer placed the order
    dine_in_type (str) : inside hotel room, dine in restaurant or parcel

  Returns
    str: A message for confirmation of food order.
  """

  if dine_in_type == "dine-in-room":
    return f"Your order have been placed. The food will get delivered at your room."
  elif dine_in_type == "dine-in-restaurant":
    return f"Your order have been placed. You will be notified once the food is almost ready."
  else:
    return f"Your order have been placed. The parcel will be ready in 45 minutes."


class requestBillingChangeRequest(BaseModel):
    complaint: str = Field(..., description="Complain about the bill. It could be that the bill is more than it should be. Or some services are charged more than it was supposed to be")
    room_number: int = Field(..., description="Which Room number the client is making billing request. If not provided, ask the user. Do not guess.")

@tool(args_schema=requestBillingChangeRequest)
def bill_complain_request(complaint: str ,room_number : int) -> str:
  """
  Handles customer information regarding billing matters, including the complaint details and his room number. Example: My bill is not correct. The food bill should be less.
  Args:
    room_number (int) : The room number from where the complain is made. Not Default value, should be asked from user.
  Returns
    str: A message for confirmation of the bill complaint.
  """

  return f"We  have received your complain {complaint}  and notified accounts department to handle the issue. Please keep your patience while we resolve. You will be notified from the front-desk once it is resolved"


# class book_room_input(BaseModel):
#     room_no: int = Field(..., description="Number of the room which need to be booked")
#     number_of_days: int = Field(description="Number of days the room needs to be booked")

# @tool(args_schema=book_room_input)
# def book_room(room_no: int, number_of_days: int) -> str:
#   """Booking a room with a room nummber and the number of days a customer will stay"""

#   if room_no == 100:
#     return "Room 100 is not available"
#   else:
#     if number_of_days > 5:
#       return f"You have to reserve two days before to stay {number_of_days} days"
#     else:
#       return f"Room {room_no} has been booked for {number_of_days} days"

tools = [get_current_temperature, book_room, order_resturant_item, bill_complain_request] # Add extra function names here...

from langchain.tools.render import format_tool_to_openai_function
functions = [format_tool_to_openai_function(f) for f in tools]

# **Model with binded functions !!**

from langchain.chat_models import ChatOpenAI
model = ChatOpenAI(openai_api_key=userdata.get('OpenAI_API_KEY')).bind(functions=functions)

# **Prompt Declaration**

from langchain.prompts import ChatPromptTemplate
prompt_model= ChatPromptTemplate.from_messages([
    ("system", "Extract the relevant information, if not explicitly provided do not guess. Extract partial info. If function not executed then answer from the {context} "),
    ("human", "{question}")
])

# **Prompt in support of chat_history**

from langchain.prompts import MessagesPlaceholder
prompt = ChatPromptTemplate.from_messages([
    MessagesPlaceholder(variable_name="chat_history"),
    prompt_model,
    MessagesPlaceholder(variable_name="agent_scratchpad"),
])

# **Creating Chain using Langchain**

from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
from langchain.schema.runnable import RunnableMap
chain = RunnableMap({
    "context": lambda x: vectorstore.similarity_search(x["question"],k=2),
    "agent_scratchpad" : lambda x: x["agent_scratchpad"],
    "chat_history" : lambda x: x["chat_history"],
    "question": lambda x: x["question"]
}) | prompt| model | OpenAIFunctionsAgentOutputParser()

# **Adding memory for conversations**

from langchain.memory import ConversationBufferMemory
memory = ConversationBufferMemory(return_messages=True,memory_key="chat_history")

# **Agent Executor for final response**

from langchain.schema.runnable import RunnablePassthrough
from langchain.agents import AgentExecutor
from langchain.agents.format_scratchpad import format_to_openai_functions

agent_chain = RunnablePassthrough.assign(
    agent_scratchpad= lambda x: format_to_openai_functions(x["intermediate_steps"]),
) | chain
agent_executor = AgentExecutor(agent=agent_chain, tools=tools, verbose=True, memory=memory)

# **Satrt Conversation with LLM for Commands & Services**

agent_executor.invoke({"question": "Hello"})

agent_executor.invoke({"question": "What is the weather in Delhi today ?"})

agent_executor.invoke({"question": "Book a room for me for the next 2"})

agent_executor.invoke({"question": "AC room"})

agent_executor.invoke({"question": "Economic class"})

agent_executor.invoke({"question": "22th February 2024 and 24th Feb 2024"})

agent_executor.invoke({"question": "+8801787440110"})

agent_executor.invoke({"question": "weather in Delhi"})

agent_executor.invoke({"question": "Who is Nasif? "})

agent_executor.invoke({"question": "What is his age? "})

agent_executor.invoke({"question": "I want to order two beer"})

agent_executor.invoke({"question": "I want it to be delivered to my room"})

agent_executor.invoke({"question": "Room 124"})

agent_executor.invoke({"question": "I have a complaint about my bill"})

agent_executor.invoke({"question": "343"})

agent_executor.invoke({"question": "It is higher than it shoud be"})

agent_executor.invoke({"question": "I am from room 543"})

agent_executor.invoke({"question": "My bill is not correct, The food bill should be less"})

Hello @salman_rakin I have similar problem . Did you find a solution for it ? Function calling in RAG. If you have a solution please share it . Thanks