Web data retrieval

import os
import asyncio
from pydantic import BaseModel
from stagehand import Stagehand, browserbase
from dotenv import load_dotenv

load_dotenv()


class Book(BaseModel):
    title: str
    price: str
    in_stock: str


class Books(BaseModel):
    books: list[Book]


async def main():
    browser = await browserbase.launch(
        api_key=os.environ["BROWSERBASE_API_KEY"]
    )
    stagehand = await Stagehand.create(browser=browser)
    page = (await browser.context.pages())[0]

    # Navigate to the site
    await page.goto("https://books.toscrape.com/")

    # Extract structured data from the page
    extract_response = await stagehand.extract(
        "Extract the books from the page including title, price, and stock status",
        Books,
    )

    print(extract_response.data)

    await stagehand.close()
    await browser.close()


if __name__ == "__main__":
    asyncio.run(main())

import os
from playwright.sync_api import sync_playwright
from browserbase import Browserbase
from dotenv import load_dotenv

load_dotenv()


def create_session():
    """Creates a Browserbase session."""
    bb = Browserbase(api_key=os.environ["BROWSERBASE_API_KEY"])
    session = bb.sessions.create(
        # Add configuration options here if needed
    )
    return session


def extract_books():
    """Extracts book data using Playwright with Browserbase."""
    session = create_session()
    print(
        f"View session replay at https://browserbase.com/sessions/{session.id}"
    )

    with sync_playwright() as p:
        browser = p.chromium.connect_over_cdp(session.connect_url)

        # Get the default browser context and page
        context = browser.contexts[0]
        page = context.pages[0]

        # Navigate to the page
        page.goto("https://books.toscrape.com/")

        # Extract the books from the page
        items = page.locator("article.product_pod")
        books = items.all()

        book_data_list = []
        for book in books:
            book_data = {
                "title": book.locator("h3 a").get_attribute("title"),
                "price": book.locator("p.price_color").text_content(),
                "image": book.locator("div.image_container img").get_attribute(
                    "src"
                ),
                "inStock": book.locator("p.instock.availability")
                .text_content()
                .strip(),
                "link": book.locator("h3 a").get_attribute("href"),
            }

            book_data_list.append(book_data)

        print("Shutting down...")
        page.close()
        browser.close()

        return book_data_list


if __name__ == "__main__":
    books = extract_books()
    print(books)