feat: Sitemap-based request list implementation (#2498)

This introduces an alternative RequestList implementation based on sitemaps. It should be possible to use this in tandem with RequestProvider in BasicCrawler, just like with the current RequestList. --------- Co-authored-by: Jindřich Bär <jindrichbar@gmail.com>

Author: janbuchar

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -14,9 +14,9 @@ import type {
  DatasetExportOptions,
  FinalStatistics,
  GetUserDataFromRequest,
+ IRequestList,
  ProxyInfo,
  Request,
- RequestList,
  RequestOptions,
  RequestProvider,
  RouterHandler,
@@ -171,7 +171,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  * > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
  * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
  */
- requestList?: RequestList;
+ requestList?: IRequestList;
 
  /**
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
@@ -445,7 +445,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  * A reference to the underlying {@apilink RequestList} class that manages the crawler's {@apilink Request|requests}.
  * Only available if used by the crawler.
  */
- requestList?: RequestList;
+ requestList?: IRequestList;
 
  /**
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
@@ -1170,7 +1170,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  * adding it back to the queue after the timeout passes. Returns `true` if the request
  * should be ignored and will be reclaimed to the queue once ready.
  */
- protected delayRequest(request: Request, source: RequestList | RequestProvider) {
+ protected delayRequest(request: Request, source: IRequestList | RequestProvider) {
  const domain = getDomain(request.url);
 
  if (!domain || !request) {
@@ -1415,7 +1415,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  protected async _requestFunctionErrorHandler(
  error: Error,
  crawlingContext: Context,
- source: RequestList | RequestProvider,
+ source: IRequestList | RequestProvider,
  ): Promise<void> {
  const { request } = crawlingContext;
  request.pushErrorMessage(error);

packages/core/src/storages/index.ts

@@ -8,3 +8,4 @@ export { RequestQueue as RequestQueueV2 } from './request_queue_v2';
 export * from './storage_manager';
 export * from './utils';
 export * from './access_checking';
+export * from './sitemap_request_list';

packages/core/src/storages/request_list.ts

@@ -20,6 +20,80 @@ export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
 
+/**
+ * Represents a static list of URLs to crawl.
+ */
+export interface IRequestList {
+ /**
+ * Returns the total number of unique requests present in the list.
+ */
+ length(): number;
+
+ /**
+ * Returns `true` if all requests were already handled and there are no more left.
+ */
+ isFinished(): Promise<boolean>;
+
+ /**
+ * Resolves to `true` if the next call to {@apilink IRequestList.fetchNextRequest} function
+ * would return `null`, otherwise it resolves to `false`.
+ * Note that even if the list is empty, there might be some pending requests currently being processed.
+ */
+ isEmpty(): Promise<boolean>;
+
+ /**
+ * Returns number of handled requests.
+ */
+ handledCount(): number;
+
+ /**
+ * Persists the current state of the `IRequestList` into the default {@apilink KeyValueStore}.
+ * The state is persisted automatically in regular intervals, but calling this method manually
+ * is useful in cases where you want to have the most current state available after you pause
+ * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
+ * a server migration.
+ */
+ persistState(): Promise<void>;
+
+ /**
+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.
+ * Otherwise it gets the next request from sources.
+ *
+ * The function's `Promise` resolves to `null` if there are no more
+ * requests to process.
+ */
+ fetchNextRequest(): Promise<Request | null>;
+
+ /**
+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.
+ * Otherwise it gets the next request from sources.
+ *
+ * The function resolves to `null` if there are no more requests to process.
+ *
+ * Can be used to iterate over the `RequestList` instance in a `for await .. of` loop.
+ * Provides an alternative for the repeated use of `fetchNextRequest`.
+ */
+ [Symbol.asyncIterator](): AsyncGenerator<Request>;
+
+ /**
+ * Reclaims request to the list if its processing failed.
+ * The request will become available in the next `this.fetchNextRequest()`.
+ */
+ reclaimRequest(request: Request): Promise<void>;
+
+ /**
+ * Marks request as handled after successful processing.
+ */
+ markRequestHandled(request: Request): Promise<void>;
+
+ /**
+ * @internal
+ */
+ inProgress: Set<string>;
+}
+
 export interface RequestListOptions {
  /**
  * An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
@@ -229,7 +303,7 @@ export interface RequestListOptions {
  * ```
  * @category Sources
  */
-export class RequestList {
+export class RequestList implements IRequestList {
  private log = log.child({ prefix: 'RequestList' });
 
  /**
@@ -431,11 +505,7 @@ export class RequestList {
  }
 
  /**
- * Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
- * The state is persisted automatically in regular intervals, but calling this method manually
- * is useful in cases where you want to have the most current state available after you pause
- * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
- * a server migration.
+ * @inheritDoc
  */
  async persistState(): Promise<void> {
  if (!this.persistStateKey) {
@@ -570,9 +640,7 @@ export class RequestList {
  }
 
  /**
- * Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
- * would return `null`, otherwise it resolves to `false`.
- * Note that even if the list is empty, there might be some pending requests currently being processed.
+ * @inheritDoc
  */
  async isEmpty(): Promise<boolean> {
  this._ensureIsInitialized();
@@ -581,7 +649,7 @@ export class RequestList {
  }
 
  /**
- * Returns `true` if all requests were already handled and there are no more left.
+ * @inheritDoc
  */
  async isFinished(): Promise<boolean> {
  this._ensureIsInitialized();
@@ -590,12 +658,7 @@ export class RequestList {
  }
 
  /**
- * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
- * using the {@apilink RequestList.reclaimRequest} function, if there is any.
- * Otherwise it gets the next request from sources.
- *
- * The function's `Promise` resolves to `null` if there are no more
- * requests to process.
+ * @inheritDoc
  */
  async fetchNextRequest(): Promise<Request | null> {
  this._ensureIsInitialized();
@@ -621,6 +684,17 @@ export class RequestList {
  return null;
  }
 
+ /**
+ * @inheritDoc
+ */
+ async *[Symbol.asyncIterator]() {
+ while (true) {
+ const req = await this.fetchNextRequest();
+ if (!req) break;
+ yield req;
+ }
+ }
+
  private ensureRequest(requestLike: Request | RequestOptions, index: number): Request {
  if (requestLike instanceof Request) {
  return requestLike;
@@ -631,7 +705,7 @@ export class RequestList {
  }
 
  /**
- * Marks request as handled after successful processing.
+ * @inheritDoc
  */
  async markRequestHandled(request: Request): Promise<void> {
  const { uniqueKey } = request;
@@ -645,8 +719,7 @@ export class RequestList {
  }
 
  /**
- * Reclaims request to the list if its processing failed.
- * The request will become available in the next `this.fetchNextRequest()`.
+ * @inheritDoc
  */
  async reclaimRequest(request: Request): Promise<void> {
  const { uniqueKey } = request;
@@ -798,7 +871,7 @@ export class RequestList {
  }
 
  /**
- * Returns number of handled requests.
+ * @inheritDoc
  */
  handledCount(): number {
  this._ensureIsInitialized();