feat: add onSkippedRequest option (#2916)

Related: #2910

Author: B4nan

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -21,6 +21,7 @@ import type {
  RouterRoutes,
  Session,
  SessionPoolOptions,
+ SkippedRequestCallback,
  Source,
  StatisticsOptions,
  StatisticState,
@@ -349,6 +350,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  */
  respectRobotsTxtFile?: boolean;
 
+ /**
+ * When a request is skipped for some reason, you can use this callback to act on it.
+ * This is currently fired only for requests skipped based on robots.txt file.
+ */
+ onSkippedRequest?: SkippedRequestCallback;
+
  /** @internal */
  log?: Log;
 
@@ -517,6 +524,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  protected httpClient: BaseHttpClient;
  protected retryOnBlocked: boolean;
  protected respectRobotsTxtFile: boolean;
+ protected onSkippedRequest?: SkippedRequestCallback;
  private _closeEvents?: boolean;
 
  private experiments: CrawlerExperiments;
@@ -552,6 +560,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
  retryOnBlocked: ow.optional.boolean,
  respectRobotsTxtFile: ow.optional.boolean,
+ onSkippedRequest: ow.optional.function,
  httpClient: ow.optional.object,
 
  // AutoscaledPool shorthands
@@ -595,6 +604,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
  retryOnBlocked = false,
  respectRobotsTxtFile = false,
+ onSkippedRequest,
 
  // internal
  log = defaultLog.child({ prefix: this.constructor.name }),
@@ -668,6 +678,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
  this.retryOnBlocked = retryOnBlocked;
  this.respectRobotsTxtFile = respectRobotsTxtFile;
+ this.onSkippedRequest = onSkippedRequest;
 
  this._handlePropertyNameChange({
  newName: 'requestHandlerTimeoutSecs',
@@ -1059,13 +1070,22 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  allowedRequests.push(request);
  } else {
  skipped.add(url);
+ await this.onSkippedRequest?.({ url, reason: 'robotsTxt' });
  }
  }
 
  if (skipped.size > 0) {
  this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
  skipped: [...skipped],
  });
+
+ if (this.onSkippedRequest) {
+ await Promise.all(
+ [...skipped].map((url) => {
+ return this.onSkippedRequest!({ url, reason: 'robotsTxt' });
+ }),
+ );
+ }
  }
 
  return requestQueue.addRequestsBatched(allowedRequests, options);
@@ -1355,12 +1375,16 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  }
 
  if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
- this.log.debug(
+ this.log.warning(
  `Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,
  );
  request.state = RequestState.SKIPPED;
  request.noRetry = true;
  await source.markRequestHandled(request);
+ await this.onSkippedRequest?.({
+ url: request.url,
+ reason: 'robotsTxt',
+ });
  return;
  }
 
@@ -1385,6 +1409,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
  // specify the RQ first to allow overriding it
  requestQueue: await this.getRequestQueue(),
  robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
+ onSkippedRequest: this.onSkippedRequest,
  ...options,
  });
  },

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -11,6 +11,7 @@ import type {
  RequestHandler,
  RequestProvider,
  Session,
+ SkippedRequestCallback,
 } from '@crawlee/basic';
 import {
  BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
@@ -626,6 +627,7 @@ export abstract class BrowserCrawler<
  page,
  requestQueue: await this.getRequestQueue(),
  robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+ onSkippedRequest: this.onSkippedRequest,
  originalRequestUrl: crawlingContext.request.url,
  finalRequestUrl: crawlingContext.request.loadedUrl,
  });
@@ -792,6 +794,7 @@ interface EnqueueLinksInternalOptions {
  page: CommonPage;
  requestQueue: RequestProvider;
  robotsTxtFile?: RobotsTxtFile;
+ onSkippedRequest?: SkippedRequestCallback;
  originalRequestUrl: string;
  finalRequestUrl?: string;
 }
@@ -802,6 +805,7 @@ export async function browserCrawlerEnqueueLinks({
  page,
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  originalRequestUrl,
  finalRequestUrl,
 }: EnqueueLinksInternalOptions) {
@@ -821,6 +825,7 @@ export async function browserCrawlerEnqueueLinks({
  return enqueueLinks({
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  urls,
  baseUrl,
  ...(options as EnqueueLinksOptions),

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -12,6 +12,7 @@ import type {
  RequestHandler,
  RequestProvider,
  RouterRoutes,
+ SkippedRequestCallback,
 } from '@crawlee/http';
 import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
@@ -194,6 +195,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
  $,
  requestQueue: await this.getRequestQueue(),
  robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+ onSkippedRequest: this.onSkippedRequest,
  originalRequestUrl: crawlingContext.request.url,
  finalRequestUrl: crawlingContext.request.loadedUrl,
  });
@@ -240,6 +242,7 @@ interface EnqueueLinksInternalOptions {
  $: cheerio.CheerioAPI | null;
  requestQueue: RequestProvider;
  robotsTxtFile?: RobotsTxtFile;
+ onSkippedRequest?: SkippedRequestCallback;
  originalRequestUrl: string;
  finalRequestUrl?: string;
 }
@@ -250,6 +253,7 @@ export async function cheerioCrawlerEnqueueLinks({
  $,
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  originalRequestUrl,
  finalRequestUrl,
 }: EnqueueLinksInternalOptions) {
@@ -273,6 +277,7 @@ export async function cheerioCrawlerEnqueueLinks({
  return enqueueLinks({
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  urls,
  baseUrl,
  ...options,

packages/core/src/enqueue_links/enqueue_links.ts

@@ -1,4 +1,4 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type RobotsTxtFile } from '@crawlee/utils';
 import ow from 'ow';
 import { getDomain } from 'tldts';
@@ -18,6 +18,8 @@ import {
  filterRequestsByPatterns,
 } from './shared';
 
+export type SkippedRequestCallback = (args: { url: string; reason: 'robotsTxt' }) => Awaitable<void>;
+
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
  /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
  limit?: number;
@@ -165,6 +167,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
  * If provided, disallowed URLs will be ignored.
  */
  robotsTxtFile?: RobotsTxtFile;
+
+ /**
+ * When a request is skipped for some reason, you can use this callback to act on it.
+ * This is currently fired only for requests skipped based on robots.txt file.
+ */
+ onSkippedRequest?: SkippedRequestCallback;
 }
 
 /**
@@ -264,6 +272,7 @@ export async function enqueueLinks(
  urls: ow.array.ofType(ow.string),
  requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
  robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
+ onSkippedRequest: ow.optional.function,
  forefront: ow.optional.boolean,
  skipNavigation: ow.optional.boolean,
  limit: ow.optional.number,
@@ -295,6 +304,7 @@ export async function enqueueLinks(
  forefront,
  waitForAllRequestsToBeAdded,
  robotsTxtFile,
+ onSkippedRequest,
  } = options;
 
  const urlExcludePatternObjects: UrlPatternObject[] = [];
@@ -373,9 +383,24 @@ export async function enqueueLinks(
  let requestOptions = createRequestOptions(urls, options);
 
  if (robotsTxtFile) {
+ const skippedRequests: RequestOptions[] = [];
+
  requestOptions = requestOptions.filter((request) => {
- return robotsTxtFile.isAllowed(request.url);
+ if (robotsTxtFile.isAllowed(request.url)) {
+ return true;
+ }
+
+ skippedRequests.push(request);
+ return false;
  });
+
+ if (onSkippedRequest && skippedRequests.length > 0) {
+ await Promise.all(
+ skippedRequests.map((request) => {
+ return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
+ }),
+ );
+ }
  }
 
  if (transformRequestFunction) {

packages/jsdom-crawler/src/internals/jsdom-crawler.ts

@@ -11,6 +11,7 @@ import type {
  RequestHandler,
  RequestProvider,
  RouterRoutes,
+ SkippedRequestCallback,
 } from '@crawlee/http';
 import {
  enqueueLinks,
@@ -305,6 +306,7 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
  window,
  requestQueue: await this.getRequestQueue(),
  robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+ onSkippedRequest: this.onSkippedRequest,
  originalRequestUrl: crawlingContext.request.url,
  finalRequestUrl: crawlingContext.request.loadedUrl,
  });
@@ -345,6 +347,7 @@ interface EnqueueLinksInternalOptions {
  window: DOMWindow | null;
  requestQueue: RequestProvider;
  robotsTxtFile?: RobotsTxtFile;
+ onSkippedRequest?: SkippedRequestCallback;
  originalRequestUrl: string;
  finalRequestUrl?: string;
 }
@@ -355,6 +358,7 @@ export async function domCrawlerEnqueueLinks({
  window,
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  originalRequestUrl,
  finalRequestUrl,
 }: EnqueueLinksInternalOptions) {
@@ -378,6 +382,7 @@ export async function domCrawlerEnqueueLinks({
  return enqueueLinks({
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  urls,
  baseUrl,
  ...options,

packages/linkedom-crawler/src/internals/linkedom-crawler.ts

@@ -10,6 +10,7 @@ import type {
  RequestHandler,
  RequestProvider,
  RouterRoutes,
+ SkippedRequestCallback,
 } from '@crawlee/http';
 import {
  enqueueLinks,
@@ -188,6 +189,7 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {
  window: document.defaultView,
  requestQueue: await this.getRequestQueue(),
  robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+ onSkippedRequest: this.onSkippedRequest,
  originalRequestUrl: crawlingContext.request.url,
  finalRequestUrl: crawlingContext.request.loadedUrl,
  });
@@ -228,6 +230,7 @@ interface EnqueueLinksInternalOptions {
  window: Window | null;
  requestQueue: RequestProvider;
  robotsTxtFile?: RobotsTxtFile;
+ onSkippedRequest?: SkippedRequestCallback;
  originalRequestUrl: string;
  finalRequestUrl?: string;
 }
@@ -238,6 +241,7 @@ export async function linkedomCrawlerEnqueueLinks({
  window,
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  originalRequestUrl,
  finalRequestUrl,
 }: EnqueueLinksInternalOptions) {
@@ -261,6 +265,7 @@ export async function linkedomCrawlerEnqueueLinks({
  return enqueueLinks({
  requestQueue,
  robotsTxtFile,
+ onSkippedRequest,
  urls,
  baseUrl,
  ...options,

test/e2e/adaptive-playwright-robots-file/actor/.actor/actor.json

@@ -0,0 +1,7 @@
+{
+"actorSpecification": 1,
+"name": "test-adaptive-playwright-robots-file",
+"version": "0.0",
+"buildTag": "latest",
+"env": null
+}

test/e2e/adaptive-playwright-robots-file/actor/.gitignore

@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage

test/e2e/adaptive-playwright-robots-file/actor/Dockerfile

@@ -0,0 +1,25 @@
+FROM node:20 AS builder
+
+COPY /packages ./packages
+COPY /package*.json ./
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional --no-audit \
+ && npm update
+
+FROM apify/actor-node-playwright-chrome:20-beta
+
+RUN rm -r node_modules
+COPY --from=builder /node_modules ./node_modules
+COPY --from=builder /packages ./packages
+COPY --from=builder /package*.json ./
+COPY /.actor ./.actor
+COPY /main.js ./
+
+RUN echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+ENV PLAYWRIGHT_EXECUTABLE_PATH=/usr/bin/google-chrome