Pagination

from scrapegraph_py import Client from pydantic import BaseModel from typing import List  class Product(BaseModel):  name: str  price: str  rating: str  class ProductList(BaseModel):  products: List[Product]  client = Client(api_key="your-api-key")  # Basic pagination - scrape 3 pages response = client.smartscraper(  website_url="https://example-store.com/products",  user_prompt="Extract all product information",  output_schema=ProductList,  total_pages=3 ) 

import { smartScraper } from 'scrapegraph-js';  const apiKey = 'your-api-key'; const url = 'https://example-store.com/products'; const prompt = 'Extract all product information';  // Basic pagination - scrape 3 pages const response = await smartScraper(  apiKey,   url,   prompt,   null,   null,   3 // total_pages ); 

from scrapegraph_py import Client from pydantic import BaseModel, Field from typing import List, Optional  class Product(BaseModel):  name: str = Field(description="Product name")  price: str = Field(description="Product price")  rating: Optional[str] = Field(description="Customer rating")  image_url: Optional[str] = Field(description="Product image URL")  availability: Optional[str] = Field(description="Product availability status")  class ProductCatalog(BaseModel):  products: List[Product] = Field(description="List of products")  client = Client(api_key="your-api-key")  # Scrape 5 pages with scrolling and delays response = client.smartscraper(  website_url="https://amazon.com/s?k=laptops",  user_prompt="Extract all laptop products with their details",  output_schema=ProductCatalog,  total_pages=5,  number_of_scrolls=3,  wait_for=2 ) 

class Article(BaseModel):  title: str = Field(description="Article title")  summary: str = Field(description="Article summary")  author: str = Field(description="Author name")  publish_date: str = Field(description="Publication date")  url: str = Field(description="Article URL")  class NewsFeed(BaseModel):  articles: List[Article] = Field(description="List of articles")  # Collect articles from multiple news pages response = client.smartscraper(  website_url="https://techcrunch.com/category/artificial-intelligence/",  user_prompt="Extract all AI-related articles with their details",  output_schema=NewsFeed,  total_pages=4,  number_of_scrolls=2 ) 

class JobListing(BaseModel):  title: str = Field(description="Job title")  company: str = Field(description="Company name")  location: str = Field(description="Job location")  salary: Optional[str] = Field(description="Salary information")  requirements: List[str] = Field(description="Job requirements")  class JobBoard(BaseModel):  jobs: List[JobListing] = Field(description="List of job listings")  # Gather job listings from multiple pages response = client.smartscraper(  website_url="https://linkedin.com/jobs/search?keywords=python",  user_prompt="Extract all Python developer job listings",  output_schema=JobBoard,  total_pages=3,  number_of_scrolls=5 ) 

# Traditional pagination (page=1, page=2, etc.) response = client.smartscraper(  website_url="https://example.com/products?page=1",  user_prompt="Extract products from this page",  total_pages=5 ) 

# Infinite scroll pagination response = client.smartscraper(  website_url="https://example.com/feed",  user_prompt="Extract all posts from the feed",  total_pages=1, # Single page  number_of_scrolls=10 # Multiple scrolls to load more content ) 

# Hybrid: multiple pages with scrolling on each response = client.smartscraper(  website_url="https://example.com/category/electronics",  user_prompt="Extract all electronic products",  total_pages=3, # 3 category pages  number_of_scrolls=5 # 5 scrolls per page ) 

# Start with 1-2 pages for testing response = client.smartscraper(  website_url="https://example.com",  user_prompt="Extract basic information",  total_pages=1 # Start small )  # Then scale up based on results response = client.smartscraper(  website_url="https://example.com",  user_prompt="Extract comprehensive data",  total_pages=5 # Scale up ) 

# Good: Specific about pagination context user_prompt = """ Extract all product information from this page and any subsequent pages. Include: name, price, rating, availability, and image URL. Ensure you capture all products across multiple pages. """  # Better: Include pagination instructions user_prompt = """ Extract product information from this e-commerce page. For each product, get: name, price, rating, availability, image URL. This is page 1 of a multi-page product listing. Look for pagination controls and extract data from all visible pages. """ 

import time  # Implement delays between requests for page in range(1, 6):  response = client.smartscraper(  website_url=f"https://example.com/products?page={page}",  user_prompt="Extract products",  total_pages=1, # One page at a time  wait_for=3 # Wait 3 seconds  )    # Additional delay between requests  time.sleep(2) 

import time from scrapegraph_py.exceptions import APIError  def scrape_with_retry(client, url, prompt, max_retries=3):  for attempt in range(max_retries):  try:  response = client.smartscraper(  website_url=url,  user_prompt=prompt,  total_pages=3  )  return response  except APIError as e:  if attempt < max_retries - 1:  print(f"Attempt {attempt + 1} failed: {e}")  time.sleep(2 ** attempt) # Exponential backoff  else:  raise e 

# Amazon product scraping response = client.smartscraper(  website_url="https://amazon.com/s?k=smartphones",  user_prompt="""  Extract all smartphone products from this search results page.  For each product include: name, price, rating, reviews count,   availability, and prime eligibility.  """,  output_schema=ProductCatalog,  total_pages=5,  number_of_scrolls=3 ) 

# Twitter/X feed scraping response = client.smartscraper(  website_url="https://twitter.com/search?q=AI",  user_prompt="""  Extract all tweets from this search results page.  For each tweet include: author, content, timestamp,   likes, retweets, and replies count.  """,  total_pages=1,  number_of_scrolls=15 # More scrolls for social media ) 

# News website scraping response = client.smartscraper(  website_url="https://reuters.com/technology",  user_prompt="""  Extract all technology news articles from this page.  For each article include: headline, summary, author,   publication date, and category.  """,  output_schema=NewsFeed,  total_pages=4 ) 

import asyncio from scrapegraph_py import AsyncClient  async def scrape_multiple_sites():  client = AsyncClient(api_key="your-api-key")    urls = [  "https://site1.com/products",  "https://site2.com/products",  "https://site3.com/products"  ]    tasks = []  for url in urls:  task = client.smartscraper(  website_url=url,  user_prompt="Extract products",  total_pages=3  )  tasks.append(task)    results = await asyncio.gather(*tasks)  return results 

def process_in_batches(urls, batch_size=3):  results = []    for i in range(0, len(urls), batch_size):  batch = urls[i:i + batch_size]    # Process batch  batch_results = []  for url in batch:  response = client.smartscraper(  website_url=url,  user_prompt="Extract data",  total_pages=2  )  batch_results.append(response)    results.extend(batch_results)    # Delay between batches  time.sleep(5)    return results