SmartScraper

from scrapegraph_py import Client  client = Client(api_key="your-api-key")  response = client.smartscraper(  website_url="https://scrapegraphai.com/",  user_prompt="Extract info about the company" ) 

import { getSmartScraperRequest } from 'scrapegraph-js';  const apiKey = 'your_api_key'; const requestId = 'ID_of_previous_request';  try {  const requestInfo = await getSmartScraperRequest(apiKey, requestId);  console.log(requestInfo); } catch (error) {  console.error(error); } 

from pydantic import BaseModel, Field  class ArticleData(BaseModel):  title: str = Field(description="Article title")  author: str = Field(description="Author name")  content: str = Field(description="Main article content")  publish_date: str = Field(description="Publication date")  response = client.smartscraper(  website_url="https://example.com/article",  user_prompt="Extract the article information",  output_schema=ArticleData ) 

import asyncio from scrapegraph_py import AsyncClient from pydantic import BaseModel, Field  # Define your schema class WebpageSchema(BaseModel):  title: str = Field(description="The title of the webpage")  description: str = Field(description="The description of the webpage")  summary: str = Field(description="A brief summary of the webpage")  async def main():  # Initialize the async client  async with AsyncClient(api_key="your-api-key") as client:  # List of URLs to analyze  urls = [  "https://scrapegraphai.com/",  "https://github.com/ScrapeGraphAI/Scrapegraph-ai",  ]   # Create scraping tasks for each URL  tasks = [  client.smartscraper(  website_url=url,  user_prompt="Summarize the main content",  output_schema=WebpageSchema  )  for url in urls  ]   # Execute requests concurrently  responses = await asyncio.gather(*tasks, return_exceptions=True)   # Process results  for i, response in enumerate(responses):  if isinstance(response, Exception):  print(f"Error for {urls[i]}: {response}")  else:  print(f"Result for {urls[i]}: {response['result']}")  # Run the async function if __name__ == "__main__":  asyncio.run(main()) 

from scrapegraph_py import Client from scrapegraph_py.logger import sgai_logger from pydantic import BaseModel from typing import List  sgai_logger.set_logging(level="INFO")  # Define the output schema class Company(BaseModel):  name: str  category: str  location: str  class CompaniesResponse(BaseModel):  companies: List[Company]  # Initialize the client with explicit API key sgai_client = Client(api_key="sgai-api-key")  try:  # SmartScraper request with infinite scroll  response = sgai_client.smartscraper(  website_url="https://www.ycombinator.com/companies?batch=Spring%202025",  user_prompt="Extract all company names and their categories from the page",  output_schema=CompaniesResponse,  number_of_scrolls=10 # Scroll 10 times to load more companies  )   # Print the response  print(f"Request ID: {response['request_id']}")    # Parse and print the results in a structured way  result = CompaniesResponse.model_validate(response['result'])  print("\nExtracted Companies:")  print("-" * 80)  for company in result.companies:  print(f"Name: {company.name}")  print(f"Category: {company.category}")  print(f"Location: {company.location}")  print("-" * 80)  except Exception as e:  print(f"An error occurred: {e}")  finally:  sgai_client.close()