Web Scraping With LangChain & Oxylabs API

python -m venv .venv
source .venv/bin/activate

pip install -U langchain-oxylabs "langchain[openai]" langgraph langchain-mcp-adapters requests python-dotenv

OXYLABS_USERNAME=your-username
OXYLABS_PASSWORD=your-password
OPENAI_API_KEY=your-openai-key

import os
from dotenv import load_dotenv
from langchain_oxylabs import OxylabsSearchAPIWrapper, OxylabsSearchRun


load_dotenv()

# Initialize the Google search tool
search = OxylabsSearchRun(
    wrapper=OxylabsSearchAPIWrapper(
        oxylabs_username=os.getenv("OXYLABS_USERNAME"),
        oxylabs_password=os.getenv("OXYLABS_PASSWORD")
    )
)

# Send a request with API parameters
response_results = search.invoke({
    "query": "best restaurants in Los Angeles",
    "geo_location": "Los Angeles,United States"
})

print(response_results)

import os
from dotenv import load_dotenv
from langgraph.prebuilt import create_react_agent
from langchain_oxylabs import OxylabsSearchAPIWrapper, OxylabsSearchRun


load_dotenv()

search = OxylabsSearchRun(
    wrapper=OxylabsSearchAPIWrapper(
        oxylabs_username=os.getenv("OXYLABS_USERNAME"),
        oxylabs_password=os.getenv("OXYLABS_PASSWORD")
    )
)

# Create an agent that uses the Google search tool
agent = create_react_agent("openai:gpt-4o-mini", [search])

user_input = "Who were the surprise performers at Coachella 2025?"

# Invoke the agent
result = agent.invoke({"messages": user_input})

# Print the agent's response
print(result["messages"][-1].content)

brew install uv

import os
import asyncio
from dotenv import load_dotenv
from langchain_mcp_adapters.sessions import create_session
from langchain_mcp_adapters.tools import load_mcp_tools
from langgraph.prebuilt import create_react_agent


load_dotenv()

# Define the MCP server config
config = {
    "transport": "stdio",
    "command": "uvx",
    "args": ["oxylabs-mcp"],
    "env": {
        "OXYLABS_USERNAME": os.getenv("OXYLABS_USERNAME"),
        "OXYLABS_PASSWORD": os.getenv("OXYLABS_PASSWORD"),
    }
}


async def main():
    # Initialize an MCP server session and load the tools
    async with create_session(config) as session:
        await session.initialize()
        tools = await load_mcp_tools(session)

        # Create an AI agent that uses the MCP server tools
        agent = create_react_agent("openai:gpt-4o-mini", tools)

        # A loop to ask questions and get answers
        while True:
            question = input("\nQuestion -> ")
            if question == "exit":
                break
            result = await agent.ainvoke({"messages": question})
            print(f"\n{result['messages'][-1].content}\n")


if __name__ == "__main__":
    asyncio.run(main())

import os
import requests
from dotenv import load_dotenv
from langchain_openai import OpenAI
from langchain_core.prompts import PromptTemplate


load_dotenv()

# Define the prompt template
prompt = PromptTemplate.from_template(
    "Analyze the following website content and summarize key points: {content}"
)

# Compose the chain using the prompt and the LLM
chain = prompt | OpenAI()


def scrape_website(url):
    """Scrape the website using Oxylabs Web Scraper API"""
    payload = {
        "source": "universal",
        "url": url,
        "parse": True
    }

    response = requests.post(
        "https://realtime.oxylabs.io/v1/queries",
        auth=(os.getenv("OXYLABS_USERNAME"), os.getenv("OXYLABS_PASSWORD")),
        json=payload
    )

    if response.status_code == 200:
        data = response.json()
        content = data["results"][0]["content"]
        print(content)
        return str(content)
    else:
        print(f"Failed to scrape website: {response.text}")
        return None


def process_content(content):
    """Process the scraped content using LangChain"""
    if not content:
        print("No content to process.")
        return None

    result = chain.invoke({"content": content})
    return result


def main(url):
    print("Scraping website...")
    scraped_content = scrape_website(url)

    if scraped_content:
        print("Processing scraped content with LangChain...")
        analysis = process_content(scraped_content)

        print("\nProcessed Analysis:\n", analysis)
    else:
        print("No content scraped.")


# Example URL to scrape
url = "https://sandbox.oxylabs.io/products/1"
main(url)

python -m venv .venv
source .venv/bin/activate

pip install -U oxylabs-ai-studio langchain "langchain[openai]" langchain-mcp-adapters langgraph

OXYLABS_AI_STUDIO_API_KEY=your_ai_studio_key
OPENAI_API_KEY=your_openai_key

from dotenv import load_dotenv
from oxylabs_ai_studio.apps.ai_crawler import AiCrawler
from langchain_openai import ChatOpenAI
from langchain.schema import HumanMessage


load_dotenv()

result = AiCrawler().crawl(
    url="https://developers.oxylabs.io/",
    user_prompt="Find all Web Scraper API targets, supported websites, and data sources.",
    output_format="markdown",
    render_javascript=False,
    return_sources_limit=10,
    geo_location="US"
)

results = result.data

llm = ChatOpenAI(model="gpt-5-nano", temperature=0)

analysis = llm.invoke([
    HumanMessage(content=f"""
    Based on this crawled content from Oxylabs documentation:
    {results}
    Please provide a structured list of all Web Scraper API targets, organized by category 
    (e.g., E-commerce, Search Engines, etc.)
    """)
])

print(analysis.content)

import os
import asyncio
from dotenv import load_dotenv
from langchain_mcp_adapters.sessions import create_session
from langchain_mcp_adapters.tools import load_mcp_tools
from langgraph.prebuilt import create_react_agent


load_dotenv()

config = {
    "transport": "stdio",
    "command": "uvx",
    "args": ["oxylabs-mcp"],
    "env": {"OXYLABS_AI_STUDIO_API_KEY": os.getenv("OXYLABS_AI_STUDIO_API_KEY")}
}


async def main():
    async with create_session(config) as session:
        await session.initialize()
        tools = await load_mcp_tools(session)

        agent = create_react_agent("openai:gpt-5-nano", tools)

        result = await agent.ainvoke(
            {
                "messages": [
                    {
                        "role": "user",
                        "content": "Search the web for the latest news on the stock market."
                    }
                ]
            }
        )
        print(f"\n{result['messages'][-1].content}\n")


if __name__ == "__main__":
    asyncio.run(main())