Handbook

import os
from dotenv import load_dotenv

load_dotenv()

# That's it! LangSmith is now tracking your agent

@tool
def search_email(query: str) -> str:
    """
    SEARCH for existing emails (does NOT send).
    Use this to FIND or LOOKUP emails.
    """

prompt = """
CRITICAL: Only use information from the provided context.
If info is not in context, say "I don't have that information."
Do NOT make up or infer information.

Context: {context}
"""

# Before: Sending entire document (10,000 tokens)
# After: Send only relevant chunks (500 tokens)

# For simple tasks
cheap_model = ChatOpenAI(model="gpt-3.5-turbo")

# For complex tasks only
expensive_model = ChatOpenAI(model="gpt-4")

# Tag your runs
from langsmith import traceable

@traceable(run_type="chain", tags=["version-1"])
def old_chain(input):
    # Old implementation
    pass

@traceable(run_type="chain", tags=["version-2"])
def new_chain(input):
    # New improved implementation
    pass

# In LangSmith, filter by tags and compare metrics

from langsmith import Client

client = Client()

# Get failed runs
failed_runs = client.list_runs(
    project_name="my-project",
    filter="error eq true",
    start_time=yesterday
)

for run in failed_runs:
    print(f"Error: {run.error}")
    print(f"Input: {run.inputs}")
    # Alert or fix automatically

from langsmith import Client

client = Client()

# Create dataset
dataset = client.create_dataset("customer-service-tests")

# Add examples
client.create_examples(
    dataset_id=dataset.id,
    inputs=[
        {"question": "What's my order status?"},
        {"question": "How do I return an item?"},
    ],
    outputs=[
        {"should_use_tool": "check_order"},
        {"should_use_tool": "return_policy"},
    ]
)

# Run tests
def test_agent():
    for example in dataset.examples:
        result = agent.invoke(example.inputs)
        # Validate result matches expected output

from langsmith import Client

client = Client()

# User gives feedback
client.create_feedback(
    run_id=run_id,
    key="helpfulness",
    score=5,  # 1-5 stars
    comment="Very helpful!"
)

# View feedback in LangSmith dashboard

import os
from dotenv import load_dotenv
from langsmith import traceable
from langchain_openai import ChatOpenAI
from langgraph.prebuilt import create_react_agent

# Enable tracing
load_dotenv()
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_PROJECT"] = "research-assistant"

# Tools with better descriptions for debugging
@traceable(name="web_search_tool")
def web_search(query: str) -> str:
    """
    Search the web for information.
    
    WHEN TO USE: User asks about current info, facts, or research topics.
    DO NOT USE: For personal info, past conversations, or saved notes.
    """
    # Implementation
    pass

@traceable(name="save_notes_tool")
def save_notes(content: str, title: str) -> str:
    """
    Save research findings to notes.
    
    WHEN TO USE: After finding important information worth remembering.
    DO NOT USE: For every small detail.
    """
    # Implementation
    pass

# Create agent with metadata
agent = create_react_agent(
    ChatOpenAI(model="gpt-4"),
    tools=[web_search, save_notes],
    state_modifier="You are a research assistant..."
)

# Track user sessions
@traceable(
    run_type="chain",
    name="research_session",
    tags=["production", "v2.0"]
)
def handle_user_query(user_id: str, query: str):
    """Handle user query with full tracing."""
    
    config = {
        "configurable": {"thread_id": user_id},
        "metadata": {
            "user_id": user_id,
            "environment": "production"
        },
        "tags": ["user_query"]
    }
    
    result = agent.invoke({"messages": [("user", query)]}, config)
    
    return result["messages"][-1].content

# Use it
response = handle_user_query("user_123", "Research quantum computing")

tags = [
    "environment:production",
    "user_type:premium",
    "feature:research",
    "version:2.1"
]

metadata = {
    "user_id": "12345",
    "session_id": "abc",
    "feature_flags": ["new_ui"],
    "model_version": "gpt-4"
}

@traceable(name="generate_report", run_type="chain")
def generate_report(data):
    # Clear name in dashboard
    pass

# 1. Environment variables set?
print(os.getenv("LANGCHAIN_TRACING_V2"))  # Should be "true"
print(os.getenv("LANGCHAIN_API_KEY"))     # Should have value

# 2. Internet connection working?

# 3. Project name correct?
print(os.getenv("LANGCHAIN_PROJECT"))

# 4. Try manual trace
from langsmith import Client
client = Client()
client.create_run(...)  # Test connection

# Only trace in development
if os.getenv("ENVIRONMENT") == "development":
    os.environ["LANGCHAIN_TRACING_V2"] = "true"

@traceable(name="process_payment", 
          hide_inputs=True,   # Don't log input
          hide_outputs=True)  # Don't log output
def process_payment(card_number, cvv):
    # Sensitive operation
    pass