Module scrapfly.scrapy.request

Classes

class ScrapflyScrapyRequest (scrape_config: ScrapeConfig,
meta: Dict = {},
*args,
**kwargs)

Expand source code

class ScrapflyScrapyRequest(Request): scrape_config: ScrapeConfig # See request_from_dict method in scrapy.utils.request attributes = tuple( attr for attr in Request.attributes if attr not in ["body", "cookies", "headers", "method", "url"]) + ( "scrape_config",) # url:str inherited # method:str inherited # body:bytes inherited # headers:Dict inherited # encoding:Dict inherited def __init__(self, scrape_config: ScrapeConfig, meta: Dict = {}, *args, **kwargs): self.scrape_config = scrape_config meta['scrapfly_scrape_config'] = self.scrape_config super().__init__( *args, url=self.scrape_config.url, headers=self.scrape_config.headers, cookies=self.scrape_config.cookies, body=self.scrape_config.body, meta=meta, **kwargs ) def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: if spider is None: raise ValueError("The 'spider' argument is required to serialize the request.") d = super().to_dict(spider=spider) d['scrape_config'] = self.scrape_config return d @classmethod def from_dict(cls, data): scrape_config_data = data['meta']['scrapfly_scrape_config'].to_dict() scrape_config = ScrapeConfig.from_dict(scrape_config_data) request = cls(scrape_config=scrape_config) return request def replace(self, *args, **kwargs): for x in [ 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs', ]: kwargs.setdefault(x, getattr(self, x)) kwargs['scrape_config'] = deepcopy(self.scrape_config) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs)

Represents an HTTP request, which is usually generated in a Spider and executed by the Downloader, thus generating a :class:Response.

Ancestors

• scrapy.http.request.Request
• scrapy.utils.trackref.object_ref

Class variables

var attributes : tuple[str, ...]

The type of the None singleton.

var scrape_config : ScrapeConfig

The type of the None singleton.

Static methods

def from_dict(data)

Methods

def replace(self, *args, **kwargs)

Expand source code

def replace(self, *args, **kwargs): for x in [ 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs', ]: kwargs.setdefault(x, getattr(self, x)) kwargs['scrape_config'] = deepcopy(self.scrape_config) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs)

Create a new Request with the same attributes except for those given new values

def to_dict(self, *, spider: ForwardRef('scrapy.Spider') | None = None) ‑> dict

Expand source code

def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: if spider is None: raise ValueError("The 'spider' argument is required to serialize the request.") d = super().to_dict(spider=spider) d['scrape_config'] = self.scrape_config return d

Return a dictionary containing the Request's data.

Use :func:~scrapy.utils.request.request_from_dict to convert back into a :class:~scrapy.Request object.

If a spider is given, this method will try to find out the name of the spider methods used as callback and errback and include them in the output dict, raising an exception if they cannot be found.