Handbook

from langchain.document_loaders import (
    TextLoader,
    PDFLoader,
    WebBaseLoader,
    DirectoryLoader
)

# Load a text file
text_loader = TextLoader("company_policy.txt")
text_docs = text_loader.load()

# Load a PDF
pdf_loader = PDFLoader("manual.pdf")
pdf_docs = pdf_loader.load()

# Load from a website
web_loader = WebBaseLoader("https://example.com/docs")
web_docs = web_loader.load()

# Load all files in a directory
dir_loader = DirectoryLoader("./documents", glob="**/*.txt")
all_docs = dir_loader.load()

from langchain.text_splitter import RecursiveCharacterTextSplitter

# Load documents
loader = TextLoader("big_document.txt")
documents = loader.load()

# Split into chunks
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,      # ~500 characters per chunk
    chunk_overlap=50,    # 50 character overlap between chunks
)

chunks = text_splitter.split_documents(documents)

print(f"Split into {len(chunks)} chunks")

from langchain_openai import OpenAIEmbeddings

embeddings = OpenAIEmbeddings()

# Embed some text
text = "The quick brown fox jumps over the lazy dog"
vector = embeddings.embed_query(text)

print(f"Embedding has {len(vector)} dimensions")
# Output: Embedding has 1536 dimensions

from langchain.vectorstores import FAISS
from langchain_openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import TextLoader

# 1. Load documents
loader = TextLoader("company_handbook.txt")
documents = loader.load()

# 2. Split into chunks
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,
    chunk_overlap=50
)
chunks = text_splitter.split_documents(documents)

# 3. Create embeddings
embeddings = OpenAIEmbeddings()

# 4. Create vector store
vectorstore = FAISS.from_documents(chunks, embeddings)

print(f"Created vector store with {len(chunks)} documents")

# Turn the vector store into a retriever
retriever = vectorstore.as_retriever(
    search_type="similarity",
    search_kwargs={"k": 3}  # Return top 3 matches
)

# Test it
question = "What's the vacation policy?"
relevant_docs = retriever.invoke(question)

for doc in relevant_docs:
    print(doc.page_content)
    print("---")

from langchain_openai import ChatOpenAI
from langchain.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

# Create the prompt
template = """Answer the question based only on the following context:

Context:
{context}

Question: {question}

Answer: """

prompt = ChatPromptTemplate.from_template(template)

# Create the chain
def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

rag_chain = (
    {
        "context": retriever | format_docs,
        "question": RunnablePassthrough()
    }
    | prompt
    | ChatOpenAI(model="gpt-4")
    | StrOutputParser()
)

# Use it!
answer = rag_chain.invoke("What's our vacation policy?")
print(answer)

from langchain.document_loaders import DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import FAISS
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

# Step 1: Load all company policy documents
loader = DirectoryLoader(
    "./company_policies",
    glob="**/*.txt"
)
documents = loader.load()

print(f"Loaded {len(documents)} documents")

# Step 2: Split into chunks
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=100
)
chunks = text_splitter.split_documents(documents)

print(f"Split into {len(chunks)} chunks")

# Step 3: Create vector store
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_documents(chunks, embeddings)

# Step 4: Create retriever
retriever = vectorstore.as_retriever(
    search_type="similarity",
    search_kwargs={"k": 3}
)

# Step 5: Create RAG chain
template = """You are a helpful assistant answering questions about company policies.

Use the following context to answer the question. If you can't find the answer in the context, say "I don't have that information in the company policies."

Context:
{context}

Question: {question}

Answer:"""

prompt = ChatPromptTemplate.from_template(template)

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

rag_chain = (
    {
        "context": retriever | format_docs,
        "question": RunnablePassthrough()
    }
    | prompt
    | ChatOpenAI(model="gpt-4", temperature=0)
    | StrOutputParser()
)

# Step 6: Use it!
questions = [
    "How many vacation days do employees get?",
    "What's the work from home policy?",
    "How do I request time off?"
]

for q in questions:
    print(f"\nQ: {q}")
    print(f"A: {rag_chain.invoke(q)}")

from langchain_core.runnables import RunnableParallel

# Modified chain that returns both answer and sources
rag_chain_with_sources = RunnableParallel(
    {
        "context": retriever,
        "question": RunnablePassthrough()
    }
).assign(
    answer=lambda x: (
        prompt 
        | ChatOpenAI(model="gpt-4") 
        | StrOutputParser()
    ).invoke({
        "context": format_docs(x["context"]),
        "question": x["question"]
    })
)

# Use it
result = rag_chain_with_sources.invoke("What's the vacation policy?")

print("Answer:", result["answer"])
print("\nSources:")
for doc in result["context"]:
    print(f"- {doc.metadata.get('source', 'Unknown')}")

from langchain.retrievers import MultiQueryRetriever

retriever = MultiQueryRetriever.from_llm(
    retriever=vectorstore.as_retriever(),
    llm=ChatOpenAI(temperature=0)
)

# This will:
# 1. Generate variations of your question
# 2. Retrieve for each variation
# 3. Combine and deduplicate results

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor

compressor = LLMChainExtractor.from_llm(ChatOpenAI())
compression_retriever = ContextualCompressionRetriever(
    base_compressor=compressor,
    base_retriever=vectorstore.as_retriever()
)

# Returns shorter, more focused results

from langchain.chains import create_history_aware_retriever

# Reformulate questions based on chat history
history_retriever = create_history_aware_retriever(
    llm=ChatOpenAI(),
    retriever=vectorstore.as_retriever(),
    prompt=history_prompt
)

# User: "What's the vacation policy?"
# Bot: "15 days per year"
# User: "Can I roll them over?"  ← Uses history to understand "them" = "vacation days"

# Test if retrieval works
test_questions = [
    "vacation policy",
    "remote work",
    "sick leave"
]

for q in test_questions:
    docs = retriever.invoke(q)
    print(f"Question: {q}")
    print(f"Retrieved {len(docs)} docs")
    print(docs[0].page_content[:200])
    print("---")

retriever = vectorstore.as_retriever(
    search_type="similarity",
    search_kwargs={
        "k": 3,  # Return top 3 results
        "score_threshold": 0.7  # Only return if similarity > 0.7
    }
)

template = """Rules:
1. Only answer from the provided context
2. If information isn't in context, say "I don't know"
3. Cite your sources
4. Be concise

Context: {context}
Question: {question}
Answer:"""

# 1. Load & split
loader = TextLoader("docs.txt")
docs = loader.load()
chunks = text_splitter.split_documents(docs)

# 2. Create vector store
vectorstore = FAISS.from_documents(chunks, embeddings)

# 3. Create retriever
retriever = vectorstore.as_retriever(search_kwargs={"k": 3})

# 4. Create chain
rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | ChatOpenAI()
    | StrOutputParser()
)

# 5. Use it
answer = rag_chain.invoke("Your question")