勉強会「【書籍出版記念】LangChainから学ぶLLMを使ったアプリケーションの工夫」で使用したソースコード
import os

import openai
from dotenv import load_dotenv

load_dotenv()

openai.api_key = os.environ["OPENAI_API_KEY"]

response = openai.ChatCompletion.create(
    model="gpt-3.5-turbo",
    messages=[
        {
            "role": "user",
            "content": "Hello! I'm Oshima.",
        },
        {
            "role": "assistant",
            "content": "Nice to meet you, Oshima! How can I assist you today?",
        },
        {
            "role": "user",
            "content": "Please introduce yourself.",
        },
    ],
    temperature=0,
)

print(response)

01_openai_completions/chat_completions.py
os1ma/langchain-rag-enhancements
import os

import openai
from dotenv import load_dotenv

load_dotenv()

openai.api_key = os.environ["OPENAI_API_KEY"]

response = openai.Completion.create(
    model="gpt-3.5-turbo-instruct",
    prompt="Hello!",
    temperature=0,
)

print(response)

01_openai_completions/completions.py
os1ma/langchain-rag-enhancements
from dotenv import load_dotenv
from langchain.document_loaders import GitLoader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import FAISS

load_dotenv()

# GitHubからLangChainのリポジトリをクローンし、.mdxファイルを読み込む
# (.mdxファイルはLangChainのドキュメントのもとになるファイルの一部)
loader = GitLoader(
    clone_url="https://github.com/langchain-ai/langchain",
    repo_path="./tmp/langchain",
    branch="master",
    file_filter=lambda f: f.endswith(".mdx"),
)
raw_docs = loader.load()
print(len(raw_docs))

# ドキュメントを1000文字単位で分割（チャンク化）する
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
docs = text_splitter.split_documents(raw_docs)
print(len(docs))

# ドキュメントをベクトル化して、FAISSに読み込む
embeddings = OpenAIEmbeddings()
db = FAISS.from_documents(docs, embeddings)
db.save_local("./tmp/faiss")

02_rag_basic/load_docs.py
os1ma/langchain-rag-enhancements
import langchain
from dotenv import load_dotenv
from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS

langchain.verbose = True

load_dotenv()

# FAISSで保存されたベクトルを読み込む
embeddings = OpenAIEmbeddings()
db = FAISS.load_local("./tmp/faiss", embeddings)
retriever = db.as_retriever()

# LangChainにおけるRAGの基本である「RetrievalQA」を準備する
chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
qa_chain = RetrievalQA.from_chain_type(
    llm=chat, chain_type="stuff", retriever=retriever
)

# 「クエリに関連する文書を検索 => LLMに回答を生成させる」という流れを実行する
query = "AWSのS3からデータを読み込むためのDocumentLoaderはありますか？"
result = qa_chain.run(query)
print(result)

02_rag_basic/rag.py
os1ma/langchain-rag-enhancements
from dotenv import load_dotenv
from langchain.embeddings.openai import OpenAIEmbeddings

load_dotenv()

text = "こんにちは。私はジョンです。よろしく。"

embeddings = OpenAIEmbeddings()
vector = embeddings.embed_query(text)
print(vector)

02_rag_basic/vectorize.py
os1ma/langchain-rag-enhancements
import langchain
from dotenv import load_dotenv
from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.retrievers import BM25Retriever, EnsembleRetriever
from langchain.vectorstores import FAISS

langchain.verbose = True

load_dotenv()

# ChatGPTで生成した架空の人名とプロフィール
texts = [
    """Name: Zephyrina Bluemoon
Profile: Zephyrina Bluemoon is an astrophysicist who was awarded the Nobel Prize in Physics in 2040. His research on dark matter and multidimensional universes has led to the development of a new cosmological theory.
""",
    """Name: Quill Solstice
Profile: Quill Solstice is an internationally renowned environmental activist, working on climate change and biodiversity conservation. His initiatives have received widespread support, especially among the youth around the world.
""",
    """Name: Seraphim Vortex
Profile: Seraphim Vortex is a globally acclaimed pianist, whose performances are often described as "the voice of nature". Through her classical music, she conveys a message of environmental preservation to the world.
""",
    """Name: Eclipse Stardust
Profile: Eclipse Stardust is an AI developer known for her research in autonomous drones. Her drone technology has been used in disaster rescue and environmental surveys, saving many lives.
""",
    """Name: Celestia Rainbow
Profile: Celestia Rainbow is a world-famous novelist, and her works have been translated into more than 30 languages. Her novels, characterized by a deep understanding of humanity and delicate portrayals of the human heart, have received international acclaim.
""",
]

# 用意したデータをFAISSで検索する準備
embeddings = OpenAIEmbeddings()
db = FAISS.from_texts(texts, embeddings)
faiss_retriever = db.as_retriever(search_kwargs={"k": 1})

# 用意したデータをBM25で検索する準備
bm25_retriever = BM25Retriever.from_texts(texts, k=1)

# 2つのRetrieverを組み合わせる
ensemble_retriever = EnsembleRetriever(
    retrievers=[bm25_retriever, faiss_retriever], weights=[0.5, 0.5]
)

# 「関連する文書を検索 => LLMに回答を生成させる」を実行する「RetrievalQA」を準備
chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
qa_chain = RetrievalQA.from_chain_type(
    llm=chat, chain_type="stuff", retriever=ensemble_retriever
)

query = "Zephyrina Bluemoonさんについて教えてください。"
result = qa_chain.run(query)
print(result)

03_rag_enhancements/ensemble_retriever.py
os1ma/langchain-rag-enhancements
import langchain
from dotenv import load_dotenv
from langchain.chains import FlareChain
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.llms import OpenAI
from langchain.vectorstores import FAISS

langchain.verbose = True

load_dotenv()

# FAISSで保存されたベクトルを読み込む
embeddings = OpenAIEmbeddings()
db = FAISS.load_local("./tmp/faiss", embeddings)
retriever = db.as_retriever()

# FLAREの準備
chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
flare = FlareChain.from_llm(
    llm=chat,
    retriever=retriever,
    min_prob=0.2,
)
# 回答の生成には、logprobsが使えるCompletions APIを使う
flare.response_chain.llm = OpenAI(
    model="gpt-3.5-turbo-instruct",
    temperature=0,
    model_kwargs={"logprobs": 1},
)

query = "LangChainとは"
result = flare.run(query)
print(result)

03_rag_enhancements/flare.py
os1ma/langchain-rag-enhancements
import langchain
from dotenv import load_dotenv
from langchain.chains import HypotheticalDocumentEmbedder, RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS

langchain.debug = True

load_dotenv()

# HyDE (LLMが生成した仮説的な回答のベクトル化) の準備
base_embeddings = OpenAIEmbeddings()
chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
embeddings = HypotheticalDocumentEmbedder.from_llm(chat, base_embeddings, "web_search")

# FAISSで保存されたベクトルを読み込む
db = FAISS.load_local("./tmp/faiss", embeddings)
retriever = db.as_retriever()

# 「関連する文書を検索 => LLMに回答を生成させる」を実行する「RetrievalQA」を準備
qa_chain = RetrievalQA.from_chain_type(
    llm=chat, chain_type="stuff", retriever=retriever
)

query = "LangChainとは"
result = qa_chain.run(query)
print(result)

03_rag_enhancements/hyde.py
os1ma/langchain-rag-enhancements
import langchain
from dotenv import load_dotenv
from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.retrievers.multi_query import MultiQueryRetriever
from langchain.vectorstores import FAISS

langchain.debug = True

load_dotenv()

# FAISSで保存されたベクトルを読み込む
embeddings = OpenAIEmbeddings()
db = FAISS.load_local("./tmp/faiss", embeddings)
base_retriever = db.as_retriever()

# MultiQueryRetrieverを準備
chat = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
retriever = MultiQueryRetriever.from_llm(retriever=base_retriever, llm=chat)

# 「関連する文書を検索 => LLMに回答を生成させる」を実行する「RetrievalQA」を準備
qa_chain = RetrievalQA.from_chain_type(
    llm=chat, chain_type="stuff", retriever=retriever
)

query = "LangChainとは"
result = qa_chain.run(query)
print(result)

03_rag_enhancements/multi_query_retriever.py
os1ma/langchain-rag-enhancements