I am trying to ask questions against a multiple pdf using pinecone and openAI but I dont know how to.
The code below works for asking questions against one document. but I would like to have multiple documents to ask questions against:
# process_message.py
from flask import request
import pinecone
# from PyPDF2 import PdfReader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import ElasticVectorSearch, Pinecone, Weaviate, FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import OpenAI
import os
import json
# from constants.company import file_company_id_column, file_location_column, file_name_column
from services.files import FileFireStorage
from middleware.auth import check_authorization
import configparser
from langchain.document_loaders import UnstructuredPDFLoader, OnlinePDFLoader, PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
def process_message():
# Create a ConfigParser object and read the config.ini file
config = configparser.ConfigParser()
config.read('config.ini')
# Retrieve the value of OPENAI_API_KEY
openai_key = config.get('openai', 'OPENAI_API_KEY')
pinecone_env_key = config.get('pinecone', 'PINECONE_ENVIRONMENT')
pinecone_api_key = config.get('pinecone', 'PINECONE_API_KEY')
loader = PyPDFLoader("docs/ops.pdf")
data = loader.load()
# data = body['data'][1]['name']
# Print information about the loaded data
print(f"You have {len(data)} document(s) in your data")
print(f"There are {len(data[30].page_content)} characters in your document")
# Chunk your data up into smaller documents
text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)
texts = text_splitter.split_documents(data)
embeddings = OpenAIEmbeddings(openai_api_key=openai_key)
pinecone.init(api_key=pinecone_api_key, environment=pinecone_env_key)
index_name = "pdf-chatbot" # Put in the name of your Pinecone index here
docsearch = Pinecone.from_texts([t.page_content for t in texts], embeddings, index_name=index_name)
# Query those docs to get your answer back
llm = OpenAI(temperature=0, openai_api_key=openai_key)
chain = load_qa_chain(llm, chain_type="stuff")
query = "Are there any other documents listed in this document?"
docs = docsearch.similarity_search(query)
answer = chain.run(input_documents=docs, question=query)
print(answer)
return answer
I added as many comments as I could there. I got this information from https://www.youtube.com/watch?v=h0DHDp1FbmQ
I tried to look at other stackoverflow questions about this but could not find anything similar
You can load multiple PDFS with PyPDFDirectoryLoader